diff options
Diffstat (limited to 'runtime')
76 files changed, 10605 insertions, 3734 deletions
diff --git a/runtime/Makefile.am b/runtime/Makefile.am index bc03c4a7..9047c83d 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -9,12 +9,14 @@ librsyslog_la_SOURCES = \ rsyslog.h \ unicode-helper.h \ atomic.h \ + batch.h \ syslogd-types.h \ module-template.h \ obj-types.h \ nsd.h \ glbl.h \ glbl.c \ + unlimited_select.h \ conf.c \ conf.h \ parser.h \ @@ -39,6 +41,8 @@ librsyslog_la_SOURCES = \ obj.h \ modules.c \ modules.h \ + apc.c \ + apc.h \ sync.c \ sync.h \ expr.c \ @@ -67,6 +71,12 @@ librsyslog_la_SOURCES = \ vmop.h \ queue.c \ queue.h \ + ruleset.c \ + ruleset.h \ + rule.c \ + rule.h \ + prop.c \ + prop.h \ cfsysline.c \ cfsysline.h \ \ @@ -105,6 +115,17 @@ lmregexp_la_LDFLAGS = -module -avoid-version lmregexp_la_LIBADD = endif +# +# zlib support +# +if ENABLE_ZLIB +pkglib_LTLIBRARIES += lmzlibw.la +lmzlibw_la_SOURCES = zlibw.c zlibw.h +lmzlibw_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmzlibw_la_LDFLAGS = -module -avoid-version +lmzlibw_la_LIBADD = +endif + if ENABLE_INET pkglib_LTLIBRARIES += lmnet.la lmnetstrms.la # @@ -116,16 +137,28 @@ lmnet_la_LDFLAGS = -module -avoid-version lmnet_la_LIBADD = # network stream master class and stream factory -lmnetstrms_la_SOURCES = netstrms.c netstrms.h netstrm.c netstrm.h nssel.c nssel.h +lmnetstrms_la_SOURCES = netstrms.c netstrms.h \ + netstrm.c netstrm.h \ + nssel.c nssel.h \ + nspoll.c nspoll.h lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version lmnetstrms_la_LIBADD = +# generic stream server framework +pkglib_LTLIBRARIES += lmstrmsrv.la +lmstrmsrv_la_SOURCES = strmsrv.c strmsrv.h strms_sess.c strms_sess.h +lmstrmsrv_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmstrmsrv_la_LDFLAGS = -module -avoid-version +lmstrmsrv_la_LIBADD = + # netstream drivers # plain tcp driver - main driver pkglib_LTLIBRARIES += lmnsd_ptcp.la -lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h nsdsel_ptcp.c nsdsel_ptcp.h +lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h \ + nsdsel_ptcp.c nsdsel_ptcp.h \ + nsdpoll_ptcp.c nsdpoll_ptcp.h lmnsd_ptcp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnsd_ptcp_la_LDFLAGS = -module -avoid-version lmnsd_ptcp_la_LIBADD = diff --git a/runtime/apc.c b/runtime/apc.c new file mode 100644 index 00000000..3c6b7ec4 --- /dev/null +++ b/runtime/apc.c @@ -0,0 +1,402 @@ +/* apc.c - asynchronous procedure call support + * + * An asynchronous procedure call (APC) is a procedure call (guess what) that is potentially run + * asynchronously to its main thread. It can be scheduled to occur at a caller-provided time. + * As long as the procedure has not been called, the APC entry may be modified by the caller + * or deleted. It is the caller's purpose to make sure proper synchronization is in place. + * The APC object only case about APC's own control structures (which *are* properly + * guarded by synchronization primitives). + * + * Module begun 2009-06-15 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <pthread.h> + +#include "rsyslog.h" +#include "obj.h" +#include "apc.h" +#include "srUtils.h" +#include "datetime.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(datetime) + +/* following is a used to implement a monotonically increasing id for the apcs. That + * ID can be used to cancel an apc request. Note that the ID is generated with modulo + * arithmetic, so at some point, it will wrap. Howerver, this happens at 2^32-1 at + * earliest, so this is not considered a problem. + */ +apc_id_t apcID = 0; + +/* private data structures */ + +/* the apc list and its entries + * This is a doubly-linked list as we need to be able to do inserts + * and deletes right in the middle of the list. It is inspired by the + * Unix callout mechanism. + * Note that we support two generic caller-provided parameters as + * experience shows that at most two are often used. This causes very + * little overhead, but simplifies caller code in cases where exactly + * two parameters are needed. We hope this is a useful optimizaton. + * rgerhards, 2009-06-15 + */ +typedef struct apc_list_s { + struct apc_list_s *pNext; + struct apc_list_s *pPrev; + apc_id_t id; + apc_t *pApc; /* pointer to the APC object to be scheduled */ +} apc_list_t; + +apc_list_t *apcListRoot = NULL; +apc_list_t *apcListTail = NULL; +pthread_mutex_t listMutex; /* needs to be locked for all list operations */ + + +/* destructor for the apc object */ +BEGINobjDestruct(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(apc) +ENDobjDestruct(apc) + + +/* ------------------------------ APC list handling functions ------------------------------ */ + +/* Function that handles changes to the list root. Most importantly, this function + * needs to schedule a new timer. It is OK to call this function with an empty list. + */ +static rsRetVal +listRootChanged(void) +{ + DEFiRet; + + if(apcListRoot == NULL) + FINALIZE; + + // TODO: implement! + +finalize_it: + RETiRet; +} + + +/* insert an apc entry into the APC list. The same entry MUST NOT already be present! + */ +static rsRetVal +insertApc(apc_t *pThis, apc_id_t *pID) +{ + apc_list_t *pCurr; + apc_list_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (apc_list_t*) calloc(1, sizeof(apc_list_t))); + pNew->pApc = pThis; + pNew->id = *pID = apcID++; +dbgprintf("insert apc %p, id %ld\n", pThis, pNew->id); + + /* find right list location */ + if(apcListRoot == NULL) { + /* no need to search, list is empty */ + apcListRoot = pNew; + apcListTail = pNew; + CHKiRet(listRootChanged()); + } else { + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->pApc->ttExec > pThis->ttExec) + break; + } + + if(pCurr == NULL) { + /* insert at tail */ + pNew->pPrev = apcListTail; + apcListTail->pNext = pNew; + apcListTail = pNew; + } else { + if(pCurr == apcListRoot) { + /* new first entry */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + apcListRoot = pNew; + CHKiRet(listRootChanged()); + } else { + /* in the middle of the list */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + } + } + } + + +finalize_it: + RETiRet; +} + + +/* Delete an apc entry from the APC list. It is OK if the entry is not found, + * in this case we assume it already has been processed. + */ +static rsRetVal +deleteApc(apc_id_t id) +{ + apc_list_t *pCurr; + DEFiRet; + +dbgprintf("trying to delete apc %ld\n", id); + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->id == id) { +RUNLOG_STR("apc id found, now deleting!\n"); + if(pCurr == apcListRoot) { + apcListRoot = pCurr->pNext; + CHKiRet(listRootChanged()); + } else { + pCurr->pPrev->pNext = pCurr->pNext; + } + if(pCurr->pNext == NULL) { + apcListTail = pCurr->pPrev; + } else { + pCurr->pNext->pPrev = pCurr->pPrev; + } + free(pCurr); + pCurr = NULL; + break; + } + } + +finalize_it: + RETiRet; +} + + +/* unlist all elements up to the current timestamp. Return this as a seperate list + * to the caller. Returns an empty (NULL ptr) list if there are no such elements. + * The caller must handle that gracefully. The list is returned in the parameter. + */ +static rsRetVal +unlistCurrent(apc_list_t **ppList) +{ + apc_list_t *pCurr; + time_t tCurr; + DEFiRet; + assert(ppList != NULL); + + datetime.GetTime(&tCurr); + + if(apcListRoot == NULL || apcListRoot->pApc->ttExec > tCurr) { + *ppList = NULL; + FINALIZE; + } + + *ppList = apcListRoot; + /* now search up to which entry we need to execute */ + for(pCurr = apcListRoot ; pCurr != NULL && pCurr->pApc->ttExec <= tCurr ; pCurr = pCurr->pNext) { + /*JUST SKIP TO LAST ELEMENT*/; + } + + if(pCurr == NULL) { + /* all elements can be unlisted */ + apcListRoot = NULL; + apcListTail = NULL; + } else { + /* need to set a new root */ + pCurr->pPrev->pNext = NULL; /* terminate newly unlisted list */ + pCurr->pPrev = NULL; /* we are the new root */ + apcListRoot = pCurr; + } + +finalize_it: + RETiRet; +} + + +/* ------------------------------ END APC list handling functions ------------------------------ */ + + +/* execute all list elements that are currently scheduled for execution. We do this in two phases. + * In the first phase, we look the list mutex and move everything from the head of the queue to + * the current timestamp to a new to-be-executed list. Then we unlock the mutex and do the actual + * exec (which may take some time). + * Note that the caller is responsible for proper + * caller-level synchronization. The caller may schedule another Apc, this module must + * ensure that (and it does so by not locking the list mutex while we call the Apc). + * Note: this function "consumes" the apc_t, so it is no longer existing after this + * function returns. + */ +// TODO make static and associated with our own pthread-based timer +rsRetVal +execScheduled(void) +{ + apc_list_t *pExecList; + apc_list_t *pCurr; + apc_list_t *pNext; + DEFiRet; + + d_pthread_mutex_lock(&listMutex); + iRet = unlistCurrent(&pExecList); + d_pthread_mutex_unlock(&listMutex); + CHKiRet(iRet); + + if(pExecList != NULL) { + DBGPRINTF("running apc scheduler - we have %s to execute\n", + pExecList == NULL ? "nothing" : "something"); + } + + for(pCurr = pExecList ; pCurr != NULL ; pCurr = pNext) { +dbgprintf("executing apc list entry %p, apc %p\n", pCurr, pCurr->pApc); + pNext = pCurr->pNext; + pCurr->pApc->pProc(pCurr->pApc->param1, pCurr->pApc->param2); + apcDestruct(&pCurr->pApc); + free(pCurr); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(apc) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(apc) + + +/* ConstructionFinalizer + * Note that we use a non-standard calling interface: pID returns the current APC + * id. This is the only way to handle the situation without the need for extra + * locking. + * rgerhards, 2008-01-09 + */ +static rsRetVal +apcConstructFinalize(apc_t *pThis, apc_id_t *pID) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, apc); + assert(pID != NULL); + d_pthread_mutex_lock(&listMutex); + insertApc(pThis, pID); + d_pthread_mutex_unlock(&listMutex); + RETiRet; +} + + +/* some set methods */ +static rsRetVal +SetProcedure(apc_t *pThis, void (*pProc)(void*, void*)) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->pProc = pProc; + return RS_RET_OK; +} +static rsRetVal +SetParam1(apc_t *pThis, void *param1) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param1; + return RS_RET_OK; +} +static rsRetVal +SetParam2(apc_t *pThis, void *param2) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param2; + return RS_RET_OK; +} + + +/* cancel an Apc request, ID is provided. It is OK if the ID can not be found, this may + * happen if the Apc was executed in the mean time. So it is safe to call CancelApc() at + * any time. + */ +static rsRetVal +CancelApc(apc_id_t id) +{ + BEGINfunc + d_pthread_mutex_lock(&listMutex); + deleteApc(id); + d_pthread_mutex_unlock(&listMutex); + ENDfunc + return RS_RET_OK; +} + + +/* debugprint for the apc object */ +BEGINobjDebugPrint(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(apc) + dbgoprint((obj_t*) pThis, "APC module, currently no state info available\n"); +ENDobjDebugPrint(apc) + + +/* queryInterface function + */ +BEGINobjQueryInterface(apc) +CODESTARTobjQueryInterface(apc) + if(pIf->ifVersion != apcCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = apcConstruct; + pIf->ConstructFinalize = apcConstructFinalize; + pIf->Destruct = apcDestruct; + pIf->DebugPrint = apcDebugPrint; + pIf->CancelApc = CancelApc; + pIf->SetProcedure = SetProcedure; + pIf->SetParam1 = SetParam1; + pIf->SetParam2 = SetParam2; +finalize_it: +ENDobjQueryInterface(apc) + + +/* Exit the apc class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(apc, OBJ_IS_CORE_MODULE) /* class, version */ + objRelease(datetime, CORE_COMPONENT); + pthread_mutex_destroy(&listMutex); +ENDObjClassExit(apc) + + +/* Initialize the apc class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(apc, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, apcDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, apcConstructFinalize); + + /* do other initializations */ + pthread_mutex_init(&listMutex, NULL); +ENDObjClassInit(apc) + +/* vi:set ai: + */ diff --git a/runtime/apc.h b/runtime/apc.h new file mode 100644 index 00000000..7c679b97 --- /dev/null +++ b/runtime/apc.h @@ -0,0 +1,56 @@ +/* The apc object. + * + * See apc.c for more information. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_APC_H +#define INCLUDED_APC_H + +/* the apc object */ +typedef struct apc_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + time_t ttExec; /* when to call procedure (so far seconds...) */ + void (*pProc)(void*, void*); /* which procedure to call */ + void *param1; /* user-supplied parameters */ + void *param2; /* user-supplied parameters */ +} apc_t; + +typedef unsigned long apc_id_t; /* monotonically incrementing apc ID */ + +/* interfaces */ +BEGINinterface(apc) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(apc); + rsRetVal (*Construct)(apc_t **ppThis); + rsRetVal (*ConstructFinalize)(apc_t *pThis, apc_id_t *); + rsRetVal (*Destruct)(apc_t **ppThis); + rsRetVal (*SetProcedure)(apc_t *pThis, void (*pProc)(void*, void*)); + rsRetVal (*SetParam1)(apc_t *pThis, void *); + rsRetVal (*SetParam2)(apc_t *pThis, void *); + rsRetVal (*CancelApc)(apc_id_t); +ENDinterface(apc) +#define apcCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(apc); + +#endif /* #ifndef INCLUDED_APC_H */ diff --git a/runtime/atomic.h b/runtime/atomic.h index fdf64214..b507b769 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -41,11 +41,18 @@ * They simply came in too late. -- rgerhards, 2008-04-02 */ #ifdef HAVE_ATOMIC_BUILTINS +# define ATOMIC_SUB(data, val) __sync_fetch_and_sub(&(data), val) +# define ATOMIC_ADD(data, val) __sync_fetch_and_add(&(data), val) # define ATOMIC_INC(data) ((void) __sync_fetch_and_add(&(data), 1)) +# define ATOMIC_INC_AND_FETCH(data) __sync_fetch_and_add(&(data), 1) # define ATOMIC_DEC(data) ((void) __sync_sub_and_fetch(&(data), 1)) # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) +# define ATOMIC_STORE_0_TO_INT(data) __sync_fetch_and_and(&(data), 0) +# define ATOMIC_STORE_1_TO_INT(data) __sync_fetch_and_or(&(data), 1) +# define ATOMIC_CAS(data, oldVal, newVal) __sync_bool_compare_and_swap(&(data), (oldVal), (newVal)); +# define ATOMIC_CAS_VAL(data, oldVal, newVal) __sync_val_compare_and_swap(&(data), (oldVal), (newVal)); #else /* note that we gained parctical proof that theoretical problems DO occur * if we do not properly address them. See this blog post for details: diff --git a/runtime/batch.h b/runtime/batch.h new file mode 100644 index 00000000..2b3aa83e --- /dev/null +++ b/runtime/batch.h @@ -0,0 +1,72 @@ +/* Definition of the batch_t data structure. + * I am not sure yet if this will become a full-blown object. For now, this header just + * includes the object definition and is not accompanied by code. + * + * Copyright 2009 by Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef BATCH_H_INCLUDED +#define BATCH_H_INCLUDED + +/* enum for batch states. Actually, we violate a layer here, in that we assume that a batch is used + * for action processing. So far, this seems acceptable, the status is simply ignored inside the + * main message queue. But over time, it could potentially be useful to split the two. + * rgerhad, 2009-05-12 + */ +typedef enum { + BATCH_STATE_RDY = 0, /* object ready for processing */ + BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ + BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unknown */ + BATCH_STATE_COMM = 3, /* message successfully commited */ + BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */ +} batch_state_t; + + +/* an object inside a batch, including any information (state!) needed for it to "life". + */ +struct batch_obj_s { + obj_t *pUsrp; /* pointer to user object (most often message) */ + batch_state_t state; /* associated state */ +}; + +/* the batch + * This object is used to dequeue multiple user pointers which are than handed over + * to processing. The size of elements is fixed after queue creation, but may be + * modified by config variables (better said: queue properties). + * Note that a "user pointer" in rsyslog context so far always is a message + * object. We stick to the more generic term because queues may potentially hold + * other types of objects, too. + * rgerhards, 2009-05-12 + * Note that nElem is not necessarily equal to nElemDeq. This is the case when we + * discard some elements (because of configuration) during dequeue processing. As + * all Elements are only deleted when the batch is processed, we can not immediately + * delete them. So we need to keep their number that we can delete them when the batch + * is completed (else, the whole process does not work correctly). + */ +struct batch_s { + int nElem; /* actual number of element in this entry */ + int nElemDeq; /* actual number of elements dequeued (and thus to be deleted) - see comment above! */ + int iDoneUpTo; /* all messages below this index have state other than RDY */ + qDeqID deqID; /* ID of dequeue operation that generated this batch */ + batch_obj_t *pElem; /* batch elements */ +}; + +#endif /* #ifndef BATCH_H_INCLUDED */ diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index e1e4a6a4..184c0d87 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -462,7 +462,7 @@ getWord(uchar **pp, cstr_t **ppStrB) ASSERT(*pp != NULL); ASSERT(ppStrB != NULL); - CHKiRet(rsCStrConstruct(ppStrB)); + CHKiRet(cstrConstruct(ppStrB)); skipWhiteSpace(pp); /* skip over any whitespace */ @@ -470,9 +470,9 @@ getWord(uchar **pp, cstr_t **ppStrB) p = *pp; while(*p && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(*ppStrB, *p++)); + CHKiRet(cstrAppendChar(*ppStrB, *p++)); } - CHKiRet(rsCStrFinish(*ppStrB)); + CHKiRet(cstrFinalize(*ppStrB)); *pp = p; @@ -506,7 +506,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); pStrB = NULL; /* we got the word, now set it */ @@ -525,7 +525,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -548,7 +548,7 @@ doSyslogName(uchar **pp, rsRetVal (*pSetHdlr)(void*, int), void *pVal, syslogNam ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); /* get word */ - iNewVal = decodeSyslogName(rsCStrGetSzStr(pStrB), pNameTable); + iNewVal = decodeSyslogName(cstrGetSzStr(pStrB), pNameTable); if(pSetHdlr == NULL) { /* we should set value directly to var */ @@ -814,7 +814,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy CHKiRet(cslcConstruct(&pThis, bChainingPermitted)); CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } /* important: add to list, AFTER everything else is OK. Else * we mess up things in the error case. @@ -825,7 +825,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(llAppend(&llCmdList, pMyCmdName, (void*) pThis)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } else { /* command already exists, are we allowed to chain? */ @@ -834,7 +834,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } diff --git a/runtime/conf.c b/runtime/conf.c index 27ab8bb4..96d9dcab 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -69,13 +69,16 @@ #include "expr.h" #include "ctok.h" #include "ctok_token.h" +#include "rule.h" +#include "ruleset.h" +#include "unicode-helper.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN #endif /* forward definitions */ -static rsRetVal cfline(uchar *line, selector_t **pfCurr); +static rsRetVal cfline(uchar *line, rule_t **pfCurr); static rsRetVal processConfFile(uchar *pConfFile); @@ -87,20 +90,21 @@ DEFobjCurrIf(ctok_token) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) +DEFobjCurrIf(rule) +DEFobjCurrIf(ruleset) -static int iNbrActions; /* number of actions the running config has. Needs to be init on ReInitConf() */ +static int iNbrActions = 0; /* number of currently defined actions */ -/* The following global variables are used for building +/* The following module-global variables are used for building * tag and host selector lines during startup and config reload. * This is stored as a global variable pool because of its ease. It is * also fairly compatible with multi-threading as the stratup code must - * be run in a single thread anyways. So there can be no race conditions. These - * variables are no longer used once the configuration has been loaded (except, - * of course, during a reload). rgerhards 2005-10-18 + * be run in a single thread anyways. So there can be no race conditions. + * rgerhards 2005-10-18 */ -EHostnameCmpMode eDfltHostnameCmpMode; -cstr_t *pDfltHostnameCmp; -cstr_t *pDfltProgNameCmp; +static EHostnameCmpMode eDfltHostnameCmpMode = HN_NO_COMP; +static cstr_t *pDfltHostnameCmp = NULL; +static cstr_t *pDfltProgNameCmp = NULL; /* process a directory and include all of its files into @@ -211,7 +215,7 @@ doIncludeLine(uchar **pp, __attribute__((unused)) void* pVal) * Required by doIncludeDirectory(). */ result = glob(pattern, GLOB_MARK, NULL, &cfgFiles); - if(result != 0) { + if(result == GLOB_NOSPACE || result == GLOB_ABORTED) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); errmsg.LogError(0, RS_RET_FILE_NOT_FOUND, "error accessing config file or directory '%s': %s", @@ -392,14 +396,17 @@ finalize_it: static rsRetVal processConfFile(uchar *pConfFile) { - DEFiRet; int iLnNbr = 0; FILE *cf; - selector_t *fCurr = NULL; + rule_t *pCurrRule = NULL; uchar *p; uchar cbuf[CFGLNSIZ]; uchar *cline; int i; + int bHadAnError = 0; + uchar *pszOrgLine = NULL; + size_t lenLine; + DEFiRet; ASSERT(pConfFile != NULL); if((cf = fopen((char*)pConfFile, "r")) == NULL) { @@ -412,9 +419,12 @@ processConfFile(uchar *pConfFile) while (fgets((char*)cline, sizeof(cbuf) - (cline - cbuf), cf) != NULL) { ++iLnNbr; /* drop LF - TODO: make it better, replace fgets(), but its clean as it is */ - if(cline[strlen((char*)cline)-1] == '\n') { - cline[strlen((char*)cline) -1] = '\0'; + lenLine = ustrlen(cline); + if(cline[lenLine-1] == '\n') { + cline[lenLine-1] = '\0'; } + free(pszOrgLine); + pszOrgLine = ustrdup(cline); /* save if needed for errmsg, NULL ptr is OK */ /* check for end-of-section, comments, strip off trailing * spaces and newline character. */ @@ -428,7 +438,6 @@ processConfFile(uchar *pConfFile) * TODO: review the code at whole - this is highly suspect (but will go away * once we do the rest of RainerScript). */ - /* was: strcpy((char*)cline, (char*)p); */ for( i = 0 ; p[i] != '\0' ; ++i) { cline[i] = p[i]; } @@ -452,7 +461,7 @@ processConfFile(uchar *pConfFile) /* we now have the complete line, and are positioned at the first non-whitespace * character. So let's process it */ - if(cfline(cbuf, &fCurr) != RS_RET_OK) { + if(cfline(cbuf, &pCurrRule) != RS_RET_OK) { /* we log a message, but otherwise ignore the error. After all, the next * line can be correct. -- rgerhards, 2007-08-02 */ @@ -460,26 +469,35 @@ processConfFile(uchar *pConfFile) dbgprintf("config line NOT successfully processed\n"); snprintf((char*)szErrLoc, sizeof(szErrLoc) / sizeof(uchar), "%s, line %d", pConfFile, iLnNbr); - errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s", (char*)szErrLoc); + errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s:\"%s\"", (char*)szErrLoc, (char*)pszOrgLine); + bHadAnError = 1; } } /* we probably have one selector left to be added - so let's do that now */ - CHKiRet(selectorAddList(fCurr)); + if(pCurrRule != NULL) { + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(pCurrRule), &pCurrRule)); + } /* close the configuration file */ - (void) fclose(cf); + fclose(cf); finalize_it: if(iRet != RS_RET_OK) { char errStr[1024]; - if(fCurr != NULL) - selectorDestruct(fCurr); + if(pCurrRule != NULL) + rule.Destruct(&pCurrRule); rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("error %d processing config file '%s'; os error (if any): %s\n", iRet, pConfFile, errStr); } + + free(pszOrgLine); + + if(bHadAnError && (iRet == RS_RET_OK)) { /* a bit dirty, enhance in future releases */ + iRet = RS_RET_NONFATAL_CONFIG_ERR; + } RETiRet; } @@ -494,7 +512,7 @@ finalize_it: rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName) { uchar *p; - uchar *tplName; + uchar *tplName = NULL; cstr_t *pStrB; DEFiRet; @@ -520,23 +538,23 @@ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEn tplName = (uchar*) strdup((char*)dfltTplName); } else { /* template specified, pick it up */ - if(rsCStrConstruct(&pStrB) != RS_RET_OK) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKiRet(cstrConstruct(&pStrB)); /* now copy the string */ while(*p && *p != '#' && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &tplName, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &tplName, 0)); } - iRet = OMSRsetEntry(pOMSR, iEntry, tplName, iTplOpts); - if(iRet != RS_RET_OK) goto finalize_it; + CHKiRet(OMSRsetEntry(pOMSR, iEntry, tplName, iTplOpts)); finalize_it: + if(iRet != RS_RET_OK) + free(tplName); + *pp = p; RETiRet; @@ -551,6 +569,7 @@ finalize_it: * rgerhards, 2007-07-25 * updated to include OMSR pointer -- rgerhards, 2007-07-27 * updated to include template name -- rgerhards, 2008-03-28 + * rgerhards, 2010-01-19: file names end at the first space */ rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl) @@ -563,7 +582,7 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int pName = pFileName; i = 1; /* we start at 1 so that we reseve space for the '\0'! */ - while(*p && *p != ';' && i < MAXFNAME) { + while(*p && *p != ';' && *p != ' ' && i < MAXFNAME) { *pName++ = *p++; ++i; } @@ -582,7 +601,7 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int * rgerhards 2005-09-15 */ /* GPLv3 - stems back to sysklogd */ -static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) { uchar *p; register uchar *q; @@ -597,17 +616,17 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f ASSERT(pline != NULL); ASSERT(*pline != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(pRule, rule); dbgprintf(" - traditional PRI filter\n"); errno = 0; /* keep strerror_r() stuff out of logerror messages */ - f->f_filter_type = FILTER_PRI; + pRule->f_filter_type = FILTER_PRI; /* Note: file structure is pre-initialized to zero because it was * created with calloc()! */ for (i = 0; i <= LOG_NFACILITIES; i++) { - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } /* scan through the list of selectors */ @@ -662,32 +681,32 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f for (i = 0; i <= LOG_NFACILITIES; i++) { if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] &= ~(1<<pri); + pRule->f_filterData.f_pmask[i] &= ~(1<<pri); else - f->f_filterData.f_pmask[i] |= (1<<pri); + pRule->f_filterData.f_pmask[i] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i] &= ~(1<<i2); + pRule->f_filterData.f_pmask[i] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i] |= (1<<i2); + pRule->f_filterData.f_pmask[i] |= (1<<i2); } } } @@ -702,27 +721,27 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] &= ~(1<<pri); + pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<pri); else - f->f_filterData.f_pmask[i >> 3] |= (1<<pri); + pRule->f_filterData.f_pmask[i >> 3] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i >> 3] &= ~(1<<i2); + pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i >> 3] |= (1<<i2); + pRule->f_filterData.f_pmask[i >> 3] |= (1<<i2); } } } @@ -748,7 +767,7 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f * A pointer to that beginnig is passed back to the caller. * rgerhards 2008-01-19 */ -static rsRetVal cflineProcessIfFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessIfFilter(uchar **pline, register rule_t *f) { DEFiRet; ctok_t *tok; @@ -761,7 +780,6 @@ static rsRetVal cflineProcessIfFilter(uchar **pline, register selector_t *f) dbgprintf(" - general expression-based filter\n"); errno = 0; /* keep strerror_r() stuff out of logerror messages */ -dbgprintf("calling expression parser, pp %p ('%s')\n", *pline, *pline); f->f_filter_type = FILTER_EXPR; /* if we come to over here, pline starts with "if ". We just skip that part. */ @@ -817,10 +835,11 @@ finalize_it: * of the action part. A pointer to that beginnig is passed back to the caller. * rgerhards 2005-09-15 */ -static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) { rsParsObj *pPars; cstr_t *pCSCompOp; + cstr_t *pCSPropName; rsRetVal iRet; int iOffset; /* for compare operations */ @@ -840,12 +859,19 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) } /* read property */ - iRet = parsDelimCStr(pPars, &f->f_filterData.prop.pCSPropName, ',', 1, 1, 1); + iRet = parsDelimCStr(pPars, &pCSPropName, ',', 1, 1, 1); + if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); + rsParsDestruct(pPars); + return(iRet); + } + iRet = propNameToID(pCSPropName, &f->f_filterData.prop.propID); if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); rsParsDestruct(pPars); return(iRet); } + cstrDestruct(&pCSPropName); /* read operation */ iRet = parsDelimCStr(pPars, &pCSCompOp, ',', 1, 1, 1); @@ -919,7 +945,7 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) */ static rsRetVal cflineProcessHostSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -945,21 +971,20 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) dbgprintf("resetting BSD-like hostname filter\n"); eDfltHostnameCmpMode = HN_NO_COMP; if(pDfltHostnameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, NULL)); } } else { dbgprintf("setting BSD-like hostname filter to '%s'\n", *pline); if(pDfltHostnameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } @@ -970,7 +995,7 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) */ static rsRetVal cflineProcessTagSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -989,34 +1014,33 @@ static rsRetVal cflineProcessTagSelector(uchar **pline) if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { dbgprintf("resetting programname filter\n"); if(pDfltProgNameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, NULL)); } } else { dbgprintf("setting programname filter to '%s'\n", *pline); if(pDfltProgNameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } /* read the filter part of a configuration line and store the filter - * in the supplied selector_t + * in the supplied rule_t * rgerhards, 2007-08-01 */ -static rsRetVal cflineDoFilter(uchar **pp, selector_t *f) +static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) { DEFiRet; ASSERT(pp != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(f, rule); /* check which filter we need to pull... */ switch(**pp) { @@ -1038,6 +1062,7 @@ static rsRetVal cflineDoFilter(uchar **pp, selector_t *f) * and, if so, we copy them over. rgerhards, 2005-10-18 */ if(pDfltProgNameCmp != NULL) { +RUNLOG_STR("dflt ProgNameCmp != NULL, setting opCSProgNameComp"); CHKiRet(rsCStrConstructFromCStr(&(f->pCSProgNameComp), pDfltProgNameCmp)); } @@ -1080,7 +1105,7 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); pAction->f_ReduceRepeated = 0; } - pAction->bEnabled = 1; /* action is enabled */ + pAction->eState = ACT_STATE_RDY; /* action is enabled */ iNbrActions++; /* one more active action! */ } break; @@ -1104,17 +1129,15 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) /* Process a configuration file line in traditional "filter selector" format - * or one that builds upon this format. + * or one that builds upon this format. Note that ppRule may be a NULL pointer, + * which is valid and happens if there is no previous line (right at the start + * of the master config file!). */ -static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) +static rsRetVal +cflineClassic(uchar *p, rule_t **ppRule) { DEFiRet; action_t *pAction; - selector_t *fCurr; - - ASSERT(pfCurr != NULL); - - fCurr = *pfCurr; /* lines starting with '&' have no new filters and just add * new actions to the currently processed selector. @@ -1132,16 +1155,19 @@ static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) * selector is NULL, which means we do not need to care about it at * all. -- rgerhards, 2007-08-01 */ - CHKiRet(selectorAddList(fCurr)); - CHKiRet(selectorConstruct(&fCurr)); /* create "fresh" selector */ - CHKiRet(cflineDoFilter(&p, fCurr)); /* pull filters */ + if(*ppRule != NULL) { + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(*ppRule), ppRule)); + } + CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ + CHKiRet(rule.SetAssRuleset(*ppRule, ruleset.GetCurrent())); /* create "fresh" selector */ + CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ + CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } CHKiRet(cflineDoAction(&p, &pAction)); - CHKiRet(llAppend(&fCurr->llActList, NULL, (void*) pAction)); + CHKiRet(llAppend(&(*ppRule)->llActList, NULL, (void*) pAction)); finalize_it: - *pfCurr = fCurr; RETiRet; } @@ -1151,7 +1177,7 @@ finalize_it: * rgerhards, 2007-08-01 */ static rsRetVal -cfline(uchar *line, selector_t **pfCurr) +cfline(uchar *line, rule_t **pfCurr) { DEFiRet; @@ -1181,21 +1207,6 @@ cfline(uchar *line, selector_t **pfCurr) } -/* Reinitialize the configuration subsystem. This is a "work-around" to the fact - * that we do not yet have actual config objects. This method is to be called - * whenever a totally new config is started (which means on startup and HUP). - * Note that it MUST NOT be called for an included config file. - * rgerhards, 2008-07-28 - */ -static rsRetVal -ReInitConf(void) -{ - DEFiRet; - iNbrActions = 0; /* this is what we created the function for ;) - action count is reset */ - RETiRet; -} - - /* return the current number of active actions * rgerhards, 2008-07-28 */ @@ -1229,7 +1240,6 @@ CODESTARTobjQueryInterface(conf) pIf->doIncludeLine = doIncludeLine; pIf->cfline = cfline; pIf->processConfFile = processConfFile; - pIf->ReInitConf = ReInitConf; pIf->GetNbrActActions = GetNbrActActions; finalize_it: @@ -1241,6 +1251,15 @@ ENDobjQueryInterface(conf) */ BEGINObjClassExit(conf, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ CODESTARTObjClassExit(conf) + /* free no-longer needed module-global variables */ + if(pDfltHostnameCmp != NULL) { + rsCStrDestruct(&pDfltHostnameCmp); + } + + if(pDfltProgNameCmp != NULL) { + rsCStrDestruct(&pDfltProgNameCmp); + } + /* release objects we no longer need */ objRelease(expr, CORE_COMPONENT); objRelease(ctok, CORE_COMPONENT); @@ -1248,6 +1267,8 @@ CODESTARTObjClassExit(conf) objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); + objRelease(rule, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDObjClassExit(conf) @@ -1263,6 +1284,8 @@ BEGINAbstractObjClassInit(conf, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANG CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* TODO: make this dependcy go away! */ + CHKiRet(objUse(rule, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); ENDObjClassInit(conf) /* vi:set ai: diff --git a/runtime/conf.h b/runtime/conf.h index 2494d4dc..d85d1f82 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -35,21 +35,23 @@ BEGINinterface(conf) /* name must also be changed in ENDinterface macro! */ rsRetVal (*cfsysline)(uchar *p); rsRetVal (*doModLoad)(uchar **pp, __attribute__((unused)) void* pVal); rsRetVal (*doIncludeLine)(uchar **pp, __attribute__((unused)) void* pVal); - rsRetVal (*cfline)(uchar *line, selector_t **pfCurr); + rsRetVal (*cfline)(uchar *line, rule_t **pfCurr); rsRetVal (*processConfFile)(uchar *pConfFile); - rsRetVal (*ReInitConf)(void); rsRetVal (*GetNbrActActions)(int *); ENDinterface(conf) -#define confCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define confCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ +/* in Version 3, entry point "ReInitConf()" was removed, as we do not longer need + * to support restart-type HUP -- rgerhards, 2009-07-15 + */ /* prototypes */ PROTOTYPEObj(conf); -/* TODO: remove them below (means move the config init code) -- rgerhards, 2008-02-19 */ -extern EHostnameCmpMode eDfltHostnameCmpMode; -extern cstr_t *pDfltHostnameCmp; -extern cstr_t *pDfltProgNameCmp; +/* TODO: the following 2 need to go in conf obj interface... */ +rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); +rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); + #endif /* #ifndef INCLUDED_CONF_H */ diff --git a/runtime/ctok.c b/runtime/ctok.c index d2cd8bbd..18ddaed2 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -87,11 +87,12 @@ ctokUngetCharFromStream(ctok_t *pThis, uchar __attribute__((unused)) c) } -/* get the next character from the input "stream" (currently just a in-memory - * string...) -- rgerhards, 2008-02-19 +/* get the next character from the input "stream". Note that this version + * does NOT look for comment characters as end-of-stream, so it is suitable + * when building constant strings! -- rgerhards, 2010-03-01 */ -static rsRetVal -ctokGetCharFromStream(ctok_t *pThis, uchar *pc) +static inline rsRetVal +ctokGetCharFromStreamNoComment(ctok_t *pThis, uchar *pc) { DEFiRet; @@ -99,7 +100,7 @@ ctokGetCharFromStream(ctok_t *pThis, uchar *pc) ASSERT(pc != NULL); /* end of string or begin of comment terminates the "stream" */ - if(*pThis->pp == '\0' || *pThis->pp == '#') { + if(*pThis->pp == '\0') { ABORT_FINALIZE(RS_RET_EOS); } else { *pc = *pThis->pp; @@ -111,6 +112,28 @@ finalize_it: } +/* get the next character from the input "stream" (currently just a in-memory + * string...) -- rgerhards, 2008-02-19 + */ +static rsRetVal +ctokGetCharFromStream(ctok_t *pThis, uchar *pc) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, ctok); + ASSERT(pc != NULL); + + CHKiRet(ctokGetCharFromStreamNoComment(pThis, pc)); + /* begin of comment terminates the "stream"! */ + if(*pc == '#') { + ABORT_FINALIZE(RS_RET_EOS); + } + +finalize_it: + RETiRet; +} + + /* skip whitespace in the input "stream". * rgerhards, 2008-02-19 */ @@ -258,18 +281,18 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_MSGVAR; } - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); /* this loop is quite simple, a variable name is terminated when a non-supported * character is detected. Note that we currently permit a numerical digit as the * first char, which is not permitted by ABNF. -- rgerhards, 2009-03-10 */ while(isalpha(c) || isdigit(c) || (c == '_') || (c == '-')) { - CHKiRet(rsCStrAppendChar(pstrVal, tolower(c))); + CHKiRet(cstrAppendChar(pstrVal, tolower(c))); CHKiRet(ctokGetCharFromStream(pThis, &c)); } CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ - CHKiRet(rsCStrFinish(pstrVal)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -277,7 +300,7 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -301,25 +324,25 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_SIMPSTR; - CHKiRet(rsCStrConstruct(&pstrVal)); - CHKiRet(ctokGetCharFromStream(pThis, &c)); + CHKiRet(cstrConstruct(&pstrVal)); + CHKiRet(ctokGetCharFromStreamNoComment(pThis, &c)); /* while we are in escape mode (had a backslash), no sequence * terminates the loop. If outside, it is terminated by a single quote. */ while(bInEsc || c != '\'') { if(bInEsc) { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); bInEsc = 0; } else { if(c == '\\') { bInEsc = 1; } else { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); } } - CHKiRet(ctokGetCharFromStream(pThis, &c)); + CHKiRet(ctokGetCharFromStreamNoComment(pThis, &c)); } - CHKiRet(rsCStrFinish(pStrB)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -327,7 +350,7 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -519,8 +542,9 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) CHKiRet(ctokUngetCharFromStream(pThis, c)); pToken->tok = ctok_FUNCTION; /* fill function name */ - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); CHKiRet(rsCStrSetSzStr(pstrVal, szWord)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); } else { /* give up... */ dbgprintf("parser has an invalid word (token) '%s'\n", szWord); diff --git a/runtime/datetime.c b/runtime/datetime.c index 19e61a0a..47d7ac0e 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -49,6 +49,8 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +/* the following table of ten powers saves us some computation */ +static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; /* ------------------------------ methods ------------------------------ */ @@ -121,9 +123,28 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) t->OffsetMode = '+'; t->OffsetHour = lBias / 3600; t->OffsetMinute = lBias % 3600; + t->timeType = TIME_TYPE_RFC5424; /* we have a high precision timestamp */ } +/* A fast alternative to getCurrTime() and time() that only obtains + * a timestamp like time() does. I was told that gettimeofday(), at + * least under Linux, is much faster than time() and I could confirm + * this testing. So I created that function as a replacement. + * rgerhards, 2009-11-12 + */ +static time_t +getTime(time_t *ttSeconds) +{ + struct timeval tp; + + if(gettimeofday(&tp, NULL) == -1) + return -1; + + if(ttSeconds != NULL) + *ttSeconds = tp.tv_sec; + return tp.tv_sec; +} /******************************************************************* @@ -140,6 +161,7 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) * DO NOT PUT ANY OTHER CODE IN THIS BEGIN ... END BLOCK!!!! */ + /** * Parse a 32 bit integer number from a string. * @@ -147,18 +169,21 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) * must be positioned at the first digit. Will be updated * so that on return it points to the first character AFTER * the integer parsed. + * \param pLenStr pointer to string length, decremented on exit by + * characters processed + * Note that if an empty string (len < 1) is passed in, + * the method always returns zero. * \retval The number parsed. */ - -static int srSLMGParseInt32(uchar** ppsz) +static int srSLMGParseInt32(uchar** ppsz, int *pLenStr) { - int i; + register int i; i = 0; - while(isdigit((int) **ppsz)) - { + while(*pLenStr > 0 && isdigit((int) **ppsz)) { i = i * 10 + **ppsz - '0'; ++(*ppsz); + --(*pLenStr); } return i; @@ -170,9 +195,13 @@ static int srSLMGParseInt32(uchar** ppsz) * updates the parse pointer position. The pTime parameter * is guranteed to be updated only if a new valid timestamp * could be obtained (restriction added 2008-09-16 by rgerhards). + * This method now also checks the maximum string length it is passed. + * If a *valid* timestamp is found, the string length is decremented + * by the number of characters processed. If it is not a valid timestamp, + * the length is kept unmodified. -- rgerhards, 2009-09-23 */ static rsRetVal -ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) +ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS, int *pLenStr) { uchar *pszTS = *ppszTS; /* variables to temporarily hold time information while we parse */ @@ -187,6 +216,7 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) char OffsetMode; /* UTC offset + or - */ char OffsetHour; /* UTC offset in hours */ int OffsetMinute; /* UTC offset in minutes */ + int lenStr; /* end variables to temporarily hold time information while we parse */ DEFiRet; @@ -194,48 +224,55 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) assert(ppszTS != NULL); assert(pszTS != NULL); - year = srSLMGParseInt32(&pszTS); + lenStr = *pLenStr; + year = srSLMGParseInt32(&pszTS, &lenStr); /* We take the liberty to accept slightly malformed timestamps e.g. in * the format of 2003-9-1T1:0:0. This doesn't hurt on receiving. Of course, * with the current state of affairs, we would never run into this code * here because at postion 11, there is no "T" in such cases ;) */ - if(*pszTS++ != '-') + if(lenStr == 0 || *pszTS++ != '-') ABORT_FINALIZE(RS_RET_INVLD_TIME); - month = srSLMGParseInt32(&pszTS); + --lenStr; + month = srSLMGParseInt32(&pszTS, &lenStr); if(month < 1 || month > 12) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != '-') + if(lenStr == 0 || *pszTS++ != '-') ABORT_FINALIZE(RS_RET_INVLD_TIME); - day = srSLMGParseInt32(&pszTS); + --lenStr; + day = srSLMGParseInt32(&pszTS, &lenStr); if(day < 1 || day > 31) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != 'T') + if(lenStr == 0 || *pszTS++ != 'T') ABORT_FINALIZE(RS_RET_INVLD_TIME); + --lenStr; - hour = srSLMGParseInt32(&pszTS); + hour = srSLMGParseInt32(&pszTS, &lenStr); if(hour < 0 || hour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - minute = srSLMGParseInt32(&pszTS); + --lenStr; + minute = srSLMGParseInt32(&pszTS, &lenStr); if(minute < 0 || minute > 59) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - second = srSLMGParseInt32(&pszTS); + --lenStr; + second = srSLMGParseInt32(&pszTS, &lenStr); if(second < 0 || second > 60) ABORT_FINALIZE(RS_RET_INVLD_TIME); /* Now let's see if we have secfrac */ - if(*pszTS == '.') { + if(lenStr > 0 && *pszTS == '.') { + --lenStr; uchar *pszStart = ++pszTS; - secfrac = srSLMGParseInt32(&pszTS); + secfrac = srSLMGParseInt32(&pszTS, &lenStr); secfracPrecision = (int) (pszTS - pszStart); } else { secfracPrecision = 0; @@ -243,23 +280,27 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) } /* check the timezone */ - if(*pszTS == 'Z') - { + if(lenStr == 0) + ABORT_FINALIZE(RS_RET_INVLD_TIME); + + if(*pszTS == 'Z') { + --lenStr; pszTS++; /* eat Z */ OffsetMode = 'Z'; OffsetHour = 0; OffsetMinute = 0; } else if((*pszTS == '+') || (*pszTS == '-')) { OffsetMode = *pszTS; + --lenStr; pszTS++; - OffsetHour = srSLMGParseInt32(&pszTS); + OffsetHour = srSLMGParseInt32(&pszTS, &lenStr); if(OffsetHour < 0 || OffsetHour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - OffsetMinute = srSLMGParseInt32(&pszTS); + OffsetMinute = srSLMGParseInt32(&pszTS, &lenStr); if(OffsetMinute < 0 || OffsetMinute > 59) ABORT_FINALIZE(RS_RET_INVLD_TIME); } else { @@ -268,10 +309,12 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) } /* OK, we actually have a 3339 timestamp, so let's indicated this */ - if(*pszTS == ' ') - ++pszTS; - else - ABORT_FINALIZE(RS_RET_INVLD_TIME); + if(lenStr > 0) { + if(*pszTS != ' ') /* if it is not a space, it can not be a "good" time - 2010-02-22 rgerhards */ + ABORT_FINALIZE(RS_RET_INVLD_TIME); + ++pszTS; /* just skip past it */ + --lenStr; + } /* we had success, so update parse pointer and caller-provided timestamp */ *ppszTS = pszTS; @@ -287,6 +330,7 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) pTime->OffsetMode = OffsetMode; pTime->OffsetHour = OffsetHour; pTime->OffsetMinute = OffsetMinute; + *pLenStr = lenStr; finalize_it: RETiRet; @@ -305,9 +349,13 @@ finalize_it: * permits us to use a pre-aquired timestamp and thus avoids to do * a (costly) time() call. Thanks to David Lang for insisting on * time() call reduction ;). + * This method now also checks the maximum string length it is passed. + * If a *valid* timestamp is found, the string length is decremented + * by the number of characters processed. If it is not a valid timestamp, + * the length is kept unmodified. -- rgerhards, 2009-09-23 */ static rsRetVal -ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) +ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS, int *pLenStr) { /* variables to temporarily hold time information while we parse */ int month; @@ -317,6 +365,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) int minute; int second; /* end variables to temporarily hold time information while we parse */ + int lenStr; uchar *pszTS; DEFiRet; @@ -324,6 +373,8 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) pszTS = *ppszTS; assert(pszTS != NULL); assert(pTime != NULL); + assert(pLenStr != NULL); + lenStr = *pLenStr; /* If we look at the month (Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec), * we may see the following character sequences occur: @@ -333,6 +384,10 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) * We will use this for parsing, as it probably is the * fastest way to parse it. * + * 2009-08-17: we now do case-insensitive comparisons, as some devices obviously do not + * obey to the RFC-specified case. As we need to guess in any case, we can ignore case + * in the first place -- rgerhards + * * 2005-07-18, well sometimes it pays to be a bit more verbose, even in C... * Fixed a bug that lead to invalid detection of the data. The issue was that * we had an if(++pszTS == 'x') inside of some of the consturcts below. However, @@ -342,22 +397,26 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) * june, when it first manifested. This also lead to invalid parsing of the rest * of the message, as the time stamp was not detected to be correct. - rgerhards */ + if(lenStr < 3) + ABORT_FINALIZE(RS_RET_INVLD_TIME); + switch(*pszTS++) { + case 'j': case 'J': - if(*pszTS == 'a') { + if(*pszTS == 'a' || *pszTS == 'A') { ++pszTS; - if(*pszTS == 'n') { + if(*pszTS == 'n' || *pszTS == 'N') { ++pszTS; month = 1; } else ABORT_FINALIZE(RS_RET_INVLD_TIME); - } else if(*pszTS == 'u') { + } else if(*pszTS == 'u' || *pszTS == 'U') { ++pszTS; - if(*pszTS == 'n') { + if(*pszTS == 'n' || *pszTS == 'N') { ++pszTS; month = 6; - } else if(*pszTS == 'l') { + } else if(*pszTS == 'l' || *pszTS == 'L') { ++pszTS; month = 7; } else @@ -365,10 +424,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'f': case 'F': - if(*pszTS == 'e') { + if(*pszTS == 'e' || *pszTS == 'E') { ++pszTS; - if(*pszTS == 'b') { + if(*pszTS == 'b' || *pszTS == 'B') { ++pszTS; month = 2; } else @@ -376,13 +436,14 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'm': case 'M': - if(*pszTS == 'a') { + if(*pszTS == 'a' || *pszTS == 'A') { ++pszTS; - if(*pszTS == 'r') { + if(*pszTS == 'r' || *pszTS == 'R') { ++pszTS; month = 3; - } else if(*pszTS == 'y') { + } else if(*pszTS == 'y' || *pszTS == 'Y') { ++pszTS; month = 5; } else @@ -390,17 +451,18 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'a': case 'A': - if(*pszTS == 'p') { + if(*pszTS == 'p' || *pszTS == 'P') { ++pszTS; - if(*pszTS == 'r') { + if(*pszTS == 'r' || *pszTS == 'R') { ++pszTS; month = 4; } else ABORT_FINALIZE(RS_RET_INVLD_TIME); - } else if(*pszTS == 'u') { + } else if(*pszTS == 'u' || *pszTS == 'U') { ++pszTS; - if(*pszTS == 'g') { + if(*pszTS == 'g' || *pszTS == 'G') { ++pszTS; month = 8; } else @@ -408,10 +470,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 's': case 'S': - if(*pszTS == 'e') { + if(*pszTS == 'e' || *pszTS == 'E') { ++pszTS; - if(*pszTS == 'p') { + if(*pszTS == 'p' || *pszTS == 'P') { ++pszTS; month = 9; } else @@ -419,10 +482,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'o': case 'O': - if(*pszTS == 'c') { + if(*pszTS == 'c' || *pszTS == 'C') { ++pszTS; - if(*pszTS == 't') { + if(*pszTS == 't' || *pszTS == 'T') { ++pszTS; month = 10; } else @@ -430,10 +494,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'n': case 'N': - if(*pszTS == 'o') { + if(*pszTS == 'o' || *pszTS == 'O') { ++pszTS; - if(*pszTS == 'v') { + if(*pszTS == 'v' || *pszTS == 'V') { ++pszTS; month = 11; } else @@ -441,10 +506,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'd': case 'D': - if(*pszTS == 'e') { + if(*pszTS == 'e' || *pszTS == 'E') { ++pszTS; - if(*pszTS == 'c') { + if(*pszTS == 'c' || *pszTS == 'C') { ++pszTS; month = 12; } else @@ -456,26 +522,32 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) ABORT_FINALIZE(RS_RET_INVLD_TIME); } + lenStr -= 3; + /* done month */ - if(*pszTS++ != ' ') + if(lenStr == 0 || *pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); + --lenStr; /* we accept a slightly malformed timestamp when receiving. This is * we accept one-digit days */ - if(*pszTS == ' ') + if(*pszTS == ' ') { + --lenStr; ++pszTS; + } - day = srSLMGParseInt32(&pszTS); + day = srSLMGParseInt32(&pszTS, &lenStr); if(day < 1 || day > 31) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ' ') + if(lenStr == 0 || *pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); + --lenStr; /* time part */ - hour = srSLMGParseInt32(&pszTS); + hour = srSLMGParseInt32(&pszTS, &lenStr); if(hour > 1970 && hour < 2100) { /* if so, we assume this actually is a year. This is a format found * e.g. in Cisco devices. @@ -485,23 +557,26 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) year = hour; /* re-query the hour, this time it must be valid */ - if(*pszTS++ != ' ') + if(lenStr == 0 || *pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); - hour = srSLMGParseInt32(&pszTS); + --lenStr; + hour = srSLMGParseInt32(&pszTS, &lenStr); } if(hour < 0 || hour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - minute = srSLMGParseInt32(&pszTS); + --lenStr; + minute = srSLMGParseInt32(&pszTS, &lenStr); if(minute < 0 || minute > 59) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - second = srSLMGParseInt32(&pszTS); + --lenStr; + second = srSLMGParseInt32(&pszTS, &lenStr); if(second < 0 || second > 60) ABORT_FINALIZE(RS_RET_INVLD_TIME); @@ -509,8 +584,16 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) * invalid format, it occurs frequently enough (e.g. with Cisco devices) * to permit it as a valid case. -- rgerhards, 2008-09-12 */ - if(*pszTS++ == ':') + if(lenStr > 0 && *pszTS == ':') { ++pszTS; /* just skip past it */ + --lenStr; + } + if(lenStr > 0) { + if(*pszTS != ' ') /* if it is not a space, it can not be a "good" time - 2010-02-22 rgerhards */ + ABORT_FINALIZE(RS_RET_INVLD_TIME); + ++pszTS; /* just skip past it */ + --lenStr; + } /* we had success, so update parse pointer and caller-provided timestamp * fields we do not have are not updated in the caller's timestamp. This @@ -527,6 +610,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) pTime->second = second; pTime->secfracPrecision = 0; pTime->secfrac = 0; + *pLenStr = lenStr; finalize_it: RETiRet; @@ -545,7 +629,7 @@ finalize_it: * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) +int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf) { /* currently we do not consider localtime/utc. This may later be * added. If so, I recommend using a property replacer option @@ -554,28 +638,54 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) * rgerhards, 2007-06-26 */ assert(ts != NULL); - assert(pDst != NULL); - - if (iLenDst < 15) /* we need at least 14 bytes - 14 digits for timestamp + '\n' */ - return(0); + assert(pBuf != NULL); - return(snprintf(pDst, iLenDst, "%4.4d%2.2d%2.2d%2.2d%2.2d%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = (ts->month / 10) % 10 + '0'; + pBuf[5] = ts->month % 10 + '0'; + pBuf[6] = (ts->day / 10) % 10 + '0'; + pBuf[7] = ts->day % 10 + '0'; + pBuf[8] = (ts->hour / 10) % 10 + '0'; + pBuf[9] = ts->hour % 10 + '0'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = (ts->second / 10) % 10 + '0'; + pBuf[13] = ts->second % 10 + '0'; + pBuf[14] = '\0'; + return 15; } -int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) +int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf) { - /* see note in formatTimestampToMySQL, applies here as well */ - assert(ts != NULL); - assert(pDst != NULL); - - if (iLenDst < 21) /* we need 20 bytes + '\n' */ - return(0); + /* see note in formatTimestampToMySQL, applies here as well */ + assert(ts != NULL); + assert(pBuf != NULL); - return(snprintf(pDst, iLenDst, "%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = ' '; + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = ':'; + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = ':'; + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + pBuf[19] = '\0'; + return 19; } @@ -585,35 +695,36 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) * buffer that will receive the resulting string. The function * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. - * The buffer must be at least 10 bytes large. + * The buffer must be at least 7 bytes large. * rgerhards, 2008-06-06 */ -int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf) { - int lenRet; - char szFmtStr[64]; + int iBuf; + int power; + int secfrac; + short digit; assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf >= 10); + iBuf = 0; if(ts->secfracPrecision > 0) - { /* We must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual formating. - */ - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), "%%0%dd", ts->secfracPrecision); - lenRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->secfrac); + { + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } } else { - pBuf[0] = '0'; - pBuf[1] = '\0'; - lenRet = 1; + pBuf[iBuf++] = '0'; } + pBuf[iBuf] = '\0'; - return(lenRet); + return iBuf; } @@ -625,48 +736,73 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestamp3339(struct syslogTime *ts, char* pBuf) { - int iRet; - char szTZ[7]; /* buffer for TZ information */ + int iBuf; + int power; + int secfrac; + short digit; + BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - - if(iLenBuf < 20) - return(0); /* we NEED at least 20 bytes */ - /* do TZ information first, this is easier to take care of "Z" zone in rfc3339 */ + /* start with fixed parts */ + /* year yyyy */ + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + /* month */ + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + /* day */ + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = 'T'; + /* hour */ + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = ':'; + /* minute */ + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = ':'; + /* second */ + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + + iBuf = 19; /* points to next free entry, now it becomes dynamic! */ + + if(ts->secfracPrecision > 0) { + pBuf[iBuf++] = '.'; + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } + } + if(ts->OffsetMode == 'Z') { - szTZ[0] = 'Z'; - szTZ[1] = '\0'; + pBuf[iBuf++] = 'Z'; } else { - snprintf(szTZ, sizeof(szTZ) / sizeof(char), "%c%2.2d:%2.2d", - ts->OffsetMode, ts->OffsetHour, ts->OffsetMinute); + pBuf[iBuf++] = ts->OffsetMode; + pBuf[iBuf++] = (ts->OffsetHour / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetHour % 10 + '0'; + pBuf[iBuf++] = ':'; + pBuf[iBuf++] = (ts->OffsetMinute / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetMinute % 10 + '0'; } - if(ts->secfracPrecision > 0) - { /* we now need to include fractional seconds. While doing so, we must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual - * formating (mmmmhhh... kind of self-modifying code... ;)). - */ - char szFmtStr[64]; - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), - "%%04d-%%02d-%%02dT%%02d:%%02d:%%02d.%%0%dd%%s", - ts->secfracPrecision); - iRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, ts->secfrac, szTZ); - } - else - iRet = snprintf(pBuf, iLenBuf, - "%4.4d-%2.2d-%2.2dT%2.2d:%2.2d:%2.2d%s", - ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, szTZ); - return(iRet); + pBuf[iBuf] = '\0'; + + ENDfunc + return iBuf; } /** @@ -676,46 +812,35 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) * returns the size of the timestamp written in bytes (without * the string termnator). If 0 is returend, an error occured. */ -int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestamp3164(struct syslogTime *ts, char* pBuf) { - static char* monthNames[13] = {"ERR", "Jan", "Feb", "Mar", - "Apr", "May", "Jun", "Jul", - "Aug", "Sep", "Oct", "Nov", "Dec"}; + static char* monthNames[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + int iDay; assert(ts != NULL); assert(pBuf != NULL); - if(iLenBuf < 16) - return(0); /* we NEED 16 bytes */ - return(snprintf(pBuf, iLenBuf, "%s %2d %2.2d:%2.2d:%2.2d", - monthNames[ts->month], ts->day, ts->hour, - ts->minute, ts->second - )); + pBuf[0] = monthNames[(ts->month - 1)% 12][0]; + pBuf[1] = monthNames[(ts->month - 1) % 12][1]; + pBuf[2] = monthNames[(ts->month - 1) % 12][2]; + pBuf[3] = ' '; + iDay = (ts->day / 10) % 10; /* we need to write a space if the first digit is 0 */ + pBuf[4] = iDay ? iDay + '0' : ' '; + pBuf[5] = ts->day % 10 + '0'; + pBuf[6] = ' '; + pBuf[7] = (ts->hour / 10) % 10 + '0'; + pBuf[8] = ts->hour % 10 + '0'; + pBuf[9] = ':'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = ':'; + pBuf[13] = (ts->second / 10) % 10 + '0'; + pBuf[14] = ts->second % 10 + '0'; + pBuf[15] = '\0'; + return 16; /* traditional: number of bytes written */ } -/** - * Format a syslogTimestamp to a text format. - * The caller must provide the timestamp as well as a character - * buffer that will receive the resulting string. The function - * returns the size of the timestamp written in bytes (without - * the string termnator). If 0 is returend, an error occured. - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int formatTimestamp(struct syslogTime *ts, char* pBuf, size_t iLenBuf) -{ - assert(ts != NULL); - assert(pBuf != NULL); - - if(ts->timeType == 1) { - return(formatTimestamp3164(ts, pBuf, iLenBuf)); - } - - if(ts->timeType == 2) { - return(formatTimestamp3339(ts, pBuf, iLenBuf)); - } - return(0); -} -#endif /* queryInterface function * rgerhards, 2008-03-05 */ @@ -731,6 +856,7 @@ CODESTARTobjQueryInterface(datetime) * of course, also affects the "if" above). */ pIf->getCurrTime = getCurrTime; + pIf->GetTime = getTime; pIf->ParseTIMESTAMP3339 = ParseTIMESTAMP3339; pIf->ParseTIMESTAMP3164 = ParseTIMESTAMP3164; pIf->formatTimestampToMySQL = formatTimestampToMySQL; @@ -749,7 +875,6 @@ ENDobjQueryInterface(datetime) BEGINAbstractObjClassInit(datetime, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); - ENDObjClassInit(datetime) /* vi:set ai: diff --git a/runtime/datetime.h b/runtime/datetime.h index efb0a0af..1de3db95 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -23,8 +23,6 @@ #ifndef INCLUDED_DATETIME_H #define INCLUDED_DATETIME_H -#include "datetime.h" - /* TODO: define error codes */ #define NO_ERRCODE -1 @@ -36,15 +34,17 @@ typedef struct datetime_s { /* interfaces */ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); - rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS); - rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS); - int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst); - int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst, size_t iLenDst); - int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); - int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); - int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); + rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS, int*); + rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS, int*); + int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst); + int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst); + int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf); + int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf); + int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); + /* v3, 2009-11-12 */ + time_t (*GetTime)(time_t *ttSeconds); ENDinterface(datetime) -#define datetimeCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define datetimeCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ /* interface changes: * 1 - initial version * 2 - not compatible to 1 - bugfix required ParseTIMESTAMP3164 to accept char ** as diff --git a/runtime/debug.c b/runtime/debug.c index 4ee90226..6d82397f 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -47,10 +47,14 @@ #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif #include "rsyslog.h" #include "debug.h" #include "atomic.h" +#include "cfsysline.h" #include "obj.h" @@ -732,6 +736,8 @@ static void dbgGetThrdName(char *pszBuf, size_t lenBuf, pthread_t thrd, int bInc */ void dbgSetThrdName(uchar *pszName) { +return; + dbgThrdInfo_t *pThrd = dbgGetThrdInfo(); if(pThrd->pszThrdName != NULL) free(pThrd->pszThrdName); @@ -776,7 +782,7 @@ static void dbgCallStackPrint(dbgThrdInfo_t *pThrd) /* print all threads call stacks */ -static void dbgCallStackPrintAll(void) +void dbgCallStackPrintAll(void) { dbgThrdInfo_t *pThrd; /* stack info */ @@ -828,34 +834,22 @@ sigsegvHdlr(int signum) abort(); } -#if 1 -#pragma GCC diagnostic ignored "-Wempty-body" -/* write the debug message. This is a helper to dbgprintf and dbgoprint which - * contains common code. added 2008-09-26 rgerhards +/* actually write the debug message. This is a separate fuction because the cleanup_push/_pop + * interface otherwise is unsafe to use (generates compiler warnings at least). + * 2009-05-20 rgerhards */ -static void -dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +static inline void +do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg) { static pthread_t ptLastThrdID = 0; static int bWasNL = 0; char pszThrdName[64]; /* 64 is to be on the safe side, anything over 20 is bad... */ - char pszWriteBuf[1024]; + char pszWriteBuf[32*1024]; size_t lenWriteBuf; struct timespec t; - uchar *pszObjName = NULL; - - /* we must get the object name before we lock the mutex, because the object - * potentially calls back into us. If we locked the mutex, we would deadlock - * ourselfs. On the other hand, the GetName call needs not to be protected, as - * this thread has a valid reference. If such an object is deleted by another - * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 - */ - if(pObj != NULL) { - pszObjName = obj.GetName(pObj); - } - - pthread_mutex_lock(&mutdbgprint); - pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif /* The bWasNL handler does not really work. It works if no thread * switching occurs during non-NL messages. Else, things are messed @@ -881,7 +875,14 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(bWasNL) { if(bPrintTime) { +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%4.4ld.%9.9ld:", (long) (t.tv_sec % 10000), t.tv_nsec); if(stddbg != -1) write(stddbg, pszWriteBuf, lenWriteBuf); @@ -903,11 +904,35 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(altdbg != -1) write(altdbg, pszMsg, lenMsg); bWasNL = (pszMsg[lenMsg - 1] == '\n') ? 1 : 0; +} + +#pragma GCC diagnostic ignored "-Wempty-body" +/* write the debug message. This is a helper to dbgprintf and dbgoprint which + * contains common code. added 2008-09-26 rgerhards + */ +static void +dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +{ + uchar *pszObjName = NULL; + + /* we must get the object name before we lock the mutex, because the object + * potentially calls back into us. If we locked the mutex, we would deadlock + * ourselfs. On the other hand, the GetName call needs not to be protected, as + * this thread has a valid reference. If such an object is deleted by another + * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 + */ + if(pObj != NULL) { + pszObjName = obj.GetName(pObj); + } + + pthread_mutex_lock(&mutdbgprint); + pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); + + do_dbgprint(pszObjName, pszMsg, lenMsg); pthread_cleanup_pop(1); } #pragma GCC diagnostic warning "-Wempty-body" -#endif /* print some debug output when an object is given * This is mostly a copy of dbgprintf, but I do not know how to combine it @@ -949,7 +974,7 @@ void dbgprintf(char *fmt, ...) { va_list ap; - char pszWriteBuf[1024]; + char pszWriteBuf[32*1024]; size_t lenWriteBuf; if(!(Debug && debugging_on)) @@ -958,6 +983,16 @@ dbgprintf(char *fmt, ...) va_start(ap, fmt); lenWriteBuf = vsnprintf(pszWriteBuf, sizeof(pszWriteBuf), fmt, ap); va_end(ap); + + if(lenWriteBuf >= sizeof(pszWriteBuf)) { + /* if we need to truncate, do it in a somewhat useful way... */ + pszWriteBuf[sizeof(pszWriteBuf) - 5] = '!'; + pszWriteBuf[sizeof(pszWriteBuf) - 4] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 3] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 2] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 1] = '\n'; + lenWriteBuf = sizeof(pszWriteBuf); + } dbgprint(NULL, pszWriteBuf, lenWriteBuf); } @@ -1050,7 +1085,9 @@ int dbgEntrFunc(dbgFuncDB_t **ppFuncDB, const char *file, const char *func, int /* when we reach this point, we have a fully-initialized FuncDB! */ ATOMIC_INC(pFuncDB->nTimesCalled); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) - dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + } if(pThrd->stackPtr >= (int) (sizeof(pThrd->callStack) / sizeof(dbgFuncDB_t*))) { dbgprintf("%s:%d: %s: debug module: call stack for this thread full, suspending call tracking\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); @@ -1080,10 +1117,12 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet) dbgFuncDBPrintActiveMutexes(pFuncDB, "WARNING: mutex still owned by us as we exit function, mutex: ", pthread_self()); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) { - if(iRet == RS_RET_NO_IRET) - dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); - else - dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + if(iRet == RS_RET_NO_IRET) + dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + else + dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + } } pThrd->stackPtr = iStackPtrRestore; if(pThrd->stackPtr < 0) { @@ -1229,6 +1268,20 @@ dbgPrintNameIsInList(const uchar *pName, dbgPrintName_t *pRoot) } +/* this is a special version of malloc that fills the alloced memory with + * HIGHVALUE, as this helps to identify bugs. -- rgerhards, 2009-10-22 + */ +void * +dbgmalloc(size_t size) +{ + void *pRet; + pRet = malloc(size); + if(pRet != NULL) + memset(pRet, 0xff, size); + return pRet; +} + + /* read in the runtime options * rgerhards, 2008-02-28 */ @@ -1261,14 +1314,22 @@ dbgGetRuntimeOptions(void) "NoLogTimestamp\n" "Nostdoout\n" "filetrace=file (may be provided multiple times)\n" + "DebugOnDemand - enables debugging on USR1, but does not turn on output\n" "\nSee debug.html in your doc set or http://www.rsyslog.com for details\n"); exit(1); } else if(!strcasecmp((char*)optname, "debug")) { /* this is earlier in the process than the -d option, as such it * allows us to spit out debug messages from the very beginning. */ - Debug = 1; + Debug = DEBUG_FULL; + debugging_on = 1; + } else if(!strcasecmp((char*)optname, "debugondemand")) { + /* Enables debugging, but turns off debug output */ + Debug = DEBUG_ONDEMAND; debugging_on = 1; + dbgprintf("Note: debug on demand turned on via configuraton file, " + "use USR1 signal to activate.\n"); + debugging_on = 0; } else if(!strcasecmp((char*)optname, "logfuncflow")) { bLogFuncFlow = 1; } else if(!strcasecmp((char*)optname, "logallocfree")) { diff --git a/runtime/debug.h b/runtime/debug.h index 1375493d..c011dd2d 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -3,7 +3,7 @@ * Definitions for the debug and run-time analysis support module. * Contains a lot of macros. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -29,6 +29,11 @@ #include <pthread.h> #include "obj-types.h" +/* some settings for various debug modes */ +#define DEBUG_OFF 0 +#define DEBUG_ONDEMAND 1 +#define DEBUG_FULL 2 + /* external static data elements (some time to be replaced) */ extern int Debug; /* debug flag - read-only after startup */ extern int debugging_on; /* read-only, except on sig USR1 */ @@ -99,6 +104,7 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet); void dbgSetExecLocation(int iStackPtr, int line); void dbgSetThrdName(uchar *pszName); void dbgPrintAllDebugInfo(void); +void *dbgmalloc(size_t size); /* macros */ #define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } @@ -126,6 +132,12 @@ void dbgPrintAllDebugInfo(void); # define RUNLOG_STR(str) #endif +#ifdef MEMCHECK +# define MALLOC(x) dbgmalloc(x) +#else +# define MALLOC(x) malloc(x) +#endif + /* mutex operations */ #define MUTOP_LOCKWAIT 1 #define MUTOP_LOCK 2 @@ -134,8 +146,7 @@ void dbgPrintAllDebugInfo(void); /* debug aides */ -//#ifdef RTINST -#if 0 // temporarily removed for helgrind +#ifdef RTINST #define d_pthread_mutex_lock(x) dbgMutexLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_trylock(x) dbgMutexTryLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_unlock(x) dbgMutexUnlock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) diff --git a/runtime/glbl.c b/runtime/glbl.c index 28f14320..ac08791f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -35,8 +35,11 @@ #include "rsyslog.h" #include "obj.h" +#include "unicode-helper.h" #include "cfsysline.h" #include "glbl.h" +#include "prop.h" +#include "atomic.h" /* some defaults */ #ifndef DFLT_NETSTRM_DRVR @@ -45,6 +48,7 @@ /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(prop) /* static data * For this object, these variables are obviously what makes the "meat" of the @@ -52,13 +56,14 @@ DEFobjStaticHelpers */ static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ -static int bHUPisRestart = 1; /* should SIGHUP cause a full system restart? */ +static int bParseHOSTNAMEandTAG = 1; /* parser modification (based on startup params!) */ static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ static int option_DisallowWarning = 1; /* complain if message from disallowed sender is received */ static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ +static prop_t *propLocalHostName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup */ static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalDomain; /* our local domain name - read-only after startup */ @@ -68,6 +73,10 @@ static uchar *pszDfltNetstrmDrvr = NULL; /* module name of default netstream dri static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm driver */ static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */ static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */ +static int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ +#ifdef USE_UNLIMITED_SELECT +static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */ +#endif /* define a macro for the simple properties' set and get functions @@ -89,9 +98,9 @@ static dataType Get##nameFunc(void) \ return(nameVar); \ } +SIMP_PROP(ParseHOSTNAMEandTAG, bParseHOSTNAMEandTAG, int) SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) SIMP_PROP(PreserveFQDN, bPreserveFQDN, int) -SIMP_PROP(HUPisRestart, bHUPisRestart, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) @@ -100,6 +109,9 @@ SIMP_PROP(DisableDNS, bDisableDNS, int) SIMP_PROP(LocalDomain, LocalDomain, uchar*) SIMP_PROP(StripDomains, StripDomains, char**) SIMP_PROP(LocalHosts, LocalHosts, char**) +#ifdef USE_UNLIMITED_SELECT +SIMP_PROP(FdSetSize, iFdSetSize, int) +#endif SIMP_PROP_SET(LocalFQDNName, LocalFQDNName, uchar*) SIMP_PROP_SET(LocalHostName, LocalHostName, uchar*) @@ -113,6 +125,24 @@ SIMP_PROP_SET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) /* TO #undef SIMP_PROP_GET +/* return global input termination status + * rgerhards, 2009-07-20 + */ +static int GetGlobalInputTermState(void) +{ + return ATOMIC_FETCH_32BIT(bTerminateInputs); +} + + +/* set global termiantion state to "terminate". Note that this is a + * "once in a lifetime" action which can not be undone. -- gerhards, 2009-07-20 + */ +static void SetGlobalInputTermination(void) +{ + ATOMIC_STORE_1_TO_INT(bTerminateInputs); +} + + /* return our local hostname. if it is not set, "[localhost]" is returned */ static uchar* @@ -132,6 +162,45 @@ GetLocalHostName(void) } +/* generate the local hostname property. This must be done after the hostname info + * has been set as well as PreserveFQDN. + * rgerhards, 2009-06-30 + */ +static rsRetVal +GenerateLocalHostNameProperty(void) +{ + DEFiRet; + uchar *pszName; + + if(propLocalHostName != NULL) + prop.Destruct(&propLocalHostName); + + CHKiRet(prop.Construct(&propLocalHostName)); + if(LocalHostName == NULL) + pszName = (uchar*) "[localhost]"; + else { + if(GetPreserveFQDN() == 1) + pszName = LocalFQDNName; + else + pszName = LocalHostName; + } + CHKiRet(prop.SetString(propLocalHostName, pszName, ustrlen(pszName))); + CHKiRet(prop.ConstructFinalize(propLocalHostName)); + +finalize_it: + RETiRet; +} + + +/* return our local hostname as a string property + */ +static prop_t* +GetLocalHostNameProp(void) +{ + return(propLocalHostName); +} + + /* return the current localhost name as FQDN (requires FQDN to be set) * TODO: we should set the FQDN ourselfs in here! */ @@ -197,13 +266,17 @@ CODESTARTobjQueryInterface(glbl) * of course, also affects the "if" above). */ pIf->GetWorkDir = GetWorkDir; + pIf->GenerateLocalHostNameProperty = GenerateLocalHostNameProperty; + pIf->GetLocalHostNameProp = GetLocalHostNameProp; + pIf->SetGlobalInputTermination = SetGlobalInputTermination; + pIf->GetGlobalInputTermState = GetGlobalInputTermState; #define SIMP_PROP(name) \ pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(ParseHOSTNAMEandTAG); SIMP_PROP(PreserveFQDN); - SIMP_PROP(HUPisRestart); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); @@ -217,6 +290,9 @@ CODESTARTobjQueryInterface(glbl) SIMP_PROP(DfltNetstrmDrvrCAF) SIMP_PROP(DfltNetstrmDrvrKeyFile) SIMP_PROP(DfltNetstrmDrvrCertFile) +#ifdef USE_UNLIMITED_SELECT + SIMP_PROP(FdSetSize) +#endif #undef SIMP_PROP finalize_it: ENDobjQueryInterface(glbl) @@ -249,8 +325,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; - bHUPisRestart = 1; bPreserveFQDN = 0; +#ifdef USE_UNLIMITED_SELECT + iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); +#endif return RS_RET_OK; } @@ -262,6 +340,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a */ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + CHKiRet(objUse(prop, CORE_COMPONENT)); /* register config handlers (TODO: we need to implement a way to unregister them) */ CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, NULL, &pszWorkDir, NULL)); @@ -271,7 +350,6 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) @@ -295,6 +373,7 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(LocalHostName); if(LocalFQDNName != NULL) free(LocalFQDNName); + objRelease(prop, CORE_COMPONENT); ENDObjClassExit(glbl) /* vi:set ai: diff --git a/runtime/glbl.h b/runtime/glbl.h index 5bdf4f57..4b4bdf83 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -8,7 +8,7 @@ * Please note that there currently is no glbl.c file as we do not yet * have any implementations. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -32,6 +32,8 @@ #ifndef GLBL_H_INCLUDED #define GLBL_H_INCLUDED +#include "prop.h" + #define glblGetIOBufSize() 4096 /* size of the IO buffer, e.g. for strm class */ /* interfaces */ @@ -42,7 +44,6 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Set##name)(dataType); SIMP_PROP(MaxLine, int) SIMP_PROP(OptimizeUniProc, int) - SIMP_PROP(HUPisRestart, int) SIMP_PROP(PreserveFQDN, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) @@ -57,9 +58,28 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(DfltNetstrmDrvrCAF, uchar*) SIMP_PROP(DfltNetstrmDrvrKeyFile, uchar*) SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*) + /* added v3, 2009-06-30 */ + rsRetVal (*GenerateLocalHostNameProperty)(void); + prop_t* (*GetLocalHostNameProp)(void); + /* added v4, 2009-07-20 */ + int (*GetGlobalInputTermState)(void); + void (*SetGlobalInputTermination)(void); + /* added v5, 2009-11-03 */ + SIMP_PROP(ParseHOSTNAMEandTAG, int) + /* note: v4, v5 are already used by more recent versions, so we need to skip them! */ + /* added v6, 2009-11-16 as part of varmojfekoj's "unlimited select()" patch + * Note that it must be always present, otherwise the interface would have different + * versions depending on compile settings, what is not acceptable. + * Use this property with care, it is only truly available if UNLIMITED_SELECT is enabled + * (I did not yet further investigate the details, because that code hopefully can be removed + * at some later stage). + */ + SIMP_PROP(FdSetSize, int) + /* v7: was neeeded to mean v5+v6 - do NOT add anything else for that version! */ + /* next change is v8! */ #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ diff --git a/runtime/linkedlist.c b/runtime/linkedlist.c index 8f842e43..cc095f6e 100644 --- a/runtime/linkedlist.c +++ b/runtime/linkedlist.c @@ -398,7 +398,7 @@ rsRetVal llExecFunc(linkedList_t *pThis, rsRetVal (*pFunc)(void*, void*), void* */ llCookie = llCookiePrev; } else if (iRet != RS_RET_OK) { - goto finalize_it; + FINALIZE; } llCookiePrev = llCookie; } diff --git a/runtime/module-template.h b/runtime/module-template.h index 6f7d877c..18aad650 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -39,12 +39,16 @@ #define DEF_OMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ - DEFobjCurrIf(obj) + DEFobjCurrIf(obj) \ + static __attribute__((unused)) int bCoreSupportsBatching; #define DEF_IMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ DEFobjCurrIf(obj) #define DEF_LMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA +#define DEF_PMOD_STATIC_DATA \ + DEFobjCurrIf(obj) \ + DEF_MOD_STATIC_DATA /* Macro to define the module type. Each module can only have a single type. If @@ -64,6 +68,7 @@ static rsRetVal modGetType(eModType_t *modType) \ #define MODULE_TYPE_INPUT MODULE_TYPE(eMOD_IN) #define MODULE_TYPE_OUTPUT MODULE_TYPE(eMOD_OUT) +#define MODULE_TYPE_PARSER MODULE_TYPE(eMOD_PARSER) #define MODULE_TYPE_LIB \ DEF_LMOD_STATIC_DATA \ MODULE_TYPE(eMOD_LIB) @@ -160,6 +165,37 @@ static rsRetVal isCompatibleWithFeature(syslogFeature __attribute__((unused)) eF RETiRet;\ } + +/* beginTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINbeginTransaction \ +static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTbeginTransaction /* currently empty, but may be extended */ + +#define ENDbeginTransaction \ + RETiRet;\ +} + + +/* endTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINendTransaction \ +static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTendTransaction /* currently empty, but may be extended */ + +#define ENDendTransaction \ + RETiRet;\ +} + + /* doAction() */ #define BEGINdoAction \ @@ -324,6 +360,29 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = tryResume;\ } + +/* the following definition is queryEtryPt block that must be added + * if an output module supports the transactional interface. + * rgerhards, 2009-04-27 + */ +#define CODEqueryEtryPt_TXIF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "beginTransaction")) {\ + *pEtryPoint = beginTransaction;\ + } else if(!strcmp((char*) name, "endTransaction")) {\ + *pEtryPoint = endTransaction;\ + } + + +/* the following definition is a queryEtryPt block that must be added + * if a non-output module supports "isCompatibleWithFeature". + * rgerhards, 2009-07-20 + */ +#define CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "isCompatibleWithFeature")) {\ + *pEtryPoint = isCompatibleWithFeature;\ + } + + /* the following definition is the standard block for queryEtryPt for INPUT * modules. This can be used if no specific handling (e.g. to cover version * differences) is needed. @@ -345,6 +404,18 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ #define CODEqueryEtryPt_STD_LIB_QUERIES \ CODEqueryEtryPt_STD_MOD_QUERIES +/* the following definition is the standard block for queryEtryPt for PARSER + * modules. This can be used if no specific handling (e.g. to cover version + * differences) is needed. + */ +#define CODEqueryEtryPt_STD_PMOD_QUERIES \ + CODEqueryEtryPt_STD_MOD_QUERIES \ + else if(!strcmp((char*) name, "parse")) {\ + *pEtryPoint = parse;\ + } else if(!strcmp((char*) name, "GetParserName")) {\ + *pEtryPoint = GetParserName;\ + } + /* modInit() * This has an extra parameter, which is the specific name of the modInit * function. That is needed for built-in modules, which must have unique @@ -393,6 +464,32 @@ finalize_it:\ } +/* now come some check functions, which enable a standard way of obtaining feature + * information from the core. feat is the to-be-tested feature and featVar is a + * variable that receives the result (0-not support, 1-supported). + * This must be a macro, so that it is put into the output's code. Otherwise, we + * would need to rely on a library entry point, which is what we intend to avoid ;) + * rgerhards, 2009-04-27 + */ +#define INITChkCoreFeature(featVar, feat) \ +{ \ + rsRetVal MACRO_Ret; \ + rsRetVal (*pQueryCoreFeatureSupport)(int*, unsigned); \ + int bSupportsIt; \ + featVar = 0; \ + MACRO_Ret = pHostQueryEtryPt((uchar*)"queryCoreFeatureSupport", &pQueryCoreFeatureSupport); \ + if(MACRO_Ret == RS_RET_OK) { \ + /* found entry point, so let's see if core supports it */ \ + CHKiRet((*pQueryCoreFeatureSupport)(&bSupportsIt, feat)); \ + if(bSupportsIt) \ + featVar = 1; \ + } else if(MACRO_Ret != RS_RET_ENTRY_POINT_NOT_FOUND) { \ + ABORT_FINALIZE(MACRO_Ret); /* Something else went wrong, what is not acceptable */ \ + } \ +} + + + /* definitions for host API queries */ #define CODEmodInit_QueryRegCFSLineHdlr \ CHKiRet(pHostQueryEtryPt((uchar*)"regCfSysLineHdlr", &omsdRegCFSLineHdlr)); @@ -509,5 +606,31 @@ static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\ } +/* parse() - main entry point of parser modules + */ +#define BEGINparse \ +static rsRetVal parse(msg_t *pMsg)\ +{\ + DEFiRet; + +#define CODESTARTparse \ + assert(pMsg != NULL); + +#define ENDparse \ + RETiRet;\ +} + + +/* function to specify the parser name. This is done via a single command which + * receives a ANSI string as parameter. + */ +#define PARSER_NAME(x) \ +static rsRetVal GetParserName(uchar **ppSz)\ +{\ + *ppSz = UCHAR_CONSTANT(x);\ + return RS_RET_OK;\ +} + + /* vim:set ai: */ diff --git a/runtime/modules.c b/runtime/modules.c index 32ae659f..1af94abc 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -11,7 +11,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -57,10 +57,12 @@ #include "cfsysline.h" #include "modules.h" #include "errmsg.h" +#include "parser.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +DEFobjCurrIf(parser) /* we must ensure that only one thread at one time tries to load or unload * modules, otherwise we may see race conditions. This first came up with @@ -77,6 +79,26 @@ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ uchar *pModDir = NULL; /* read-only after startup */ +/* we provide a set of dummy functions for modules that do not support the + * some interfaces. + * On the commit feature: As the modules do not support it, they commit each message they + * receive, and as such the dummies can always return RS_RET_OK without causing + * harm. This simplifies things as in action processing we do not need to check + * if the transactional entry points exist. + */ +static rsRetVal dummyBeginTransaction() +{ + return RS_RET_OK; +} +static rsRetVal dummyEndTransaction() +{ + return RS_RET_OK; +} +static rsRetVal dummyIsCompatibleWithFeature() +{ + return RS_RET_INCOMPATIBLE; +} + #ifdef DEBUG /* we add some home-grown support to track our users (and detect who does not free us). In * the long term, this should probably be migrated into debug.c (TODO). -- rgerhards, 2008-03-11 @@ -216,19 +238,38 @@ static void moduleDestruct(modInfo_t *pThis) } +/* This enables a module to query the core for specific features. + * rgerhards, 2009-04-22 + */ +static rsRetVal queryCoreFeatureSupport(int *pBool, unsigned uFeat) +{ + DEFiRet; + + if((pBool == NULL)) + ABORT_FINALIZE(RS_RET_PARAM_ERROR); + + *pBool = (uFeat & CORE_FEATURE_BATCHING) ? 1 : 0; + +finalize_it: + RETiRet; +} + + /* The following function is the queryEntryPoint for host-based entry points. * Modules may call it to get access to core interface functions. Please note * that utility functions can be accessed via shared libraries - at least this * is my current shool of thinking. * Please note that the implementation as a query interface allows to take * care of plug-in interface version differences. -- rgerhards, 2007-07-31 + * ... but often it better not to use a new interface. So we now add core + * functions here that a plugin may request. -- rgerhards, 2009-04-22 */ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) { DEFiRet; if((name == NULL) || (pEtryPoint == NULL)) - return RS_RET_PARAM_ERROR; + ABORT_FINALIZE(RS_RET_PARAM_ERROR); if(!strcmp((char*) name, "regCfSysLineHdlr")) { *pEtryPoint = regCfSysLineHdlr; @@ -236,6 +277,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) *pEtryPoint = objGetObjInterface; } else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) { *pEtryPoint = OMSRgetSupportedTplOpts; + } else if(!strcmp((char*) name, "queryCoreFeatureSupport")) { + *pEtryPoint = queryCoreFeatureSupport; } else { *pEtryPoint = NULL; /* to be on the safe side */ ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND); @@ -361,10 +404,13 @@ finalize_it: static rsRetVal doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr) { - DEFiRet; rsRetVal localRet; modInfo_t *pNew = NULL; + uchar *pParserName; + parser_t *pParser; /* used for parser modules */ + rsRetVal (*GetParserName)(uchar**); rsRetVal (*modGetType)(eModType_t *pType); + DEFiRet; assert(modInit != NULL); @@ -383,7 +429,7 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ * can never change in the lifetime of an module. -- rgerhards, 2007-12-14 */ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getType", &modGetType)); - CHKiRet((iRet = (*modGetType)(&pNew->eType)) != RS_RET_OK); + CHKiRet((*modGetType)(&pNew->eType)); dbgprintf("module of type %d being loaded.\n", pNew->eType); /* OK, we know we can successfully work with the module. So we now fill the @@ -392,6 +438,11 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ */ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modGetID", &pNew->modGetID)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modExit", &pNew->modExit)); + localRet = (*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->isCompatibleWithFeature = dummyIsCompatibleWithFeature; + else if(localRet != RS_RET_OK) + ABORT_FINALIZE(localRet); /* ... and now the module-specific interfaces */ switch(pNew->eType) { @@ -399,21 +450,61 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"runInput", &pNew->mod.im.runInput)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"willRun", &pNew->mod.im.willRun)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"afterRun", &pNew->mod.im.afterRun)); + pNew->mod.im.bCanRun = 0; break; case eMOD_OUT: CHKiRet((*pNew->modQueryEtryPt)((uchar*)"freeInstance", &pNew->freeInstance)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"dbgPrintInstInfo", &pNew->dbgPrintInstInfo)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"doAction", &pNew->mod.om.doAction)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct)); - CHKiRet((*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume)); /* try load optional interfaces */ localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) ABORT_FINALIZE(localRet); + + localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->mod.om.beginTransaction = dummyBeginTransaction; + else if(localRet != RS_RET_OK) + ABORT_FINALIZE(localRet); + + localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) { + pNew->mod.om.endTransaction = dummyEndTransaction; + } else if(localRet != RS_RET_OK) { + ABORT_FINALIZE(localRet); + } break; case eMOD_LIB: break; + case eMOD_PARSER: + /* first, we need to obtain the parser object. We could not do that during + * init as that would have caused class bootstrap issues which are not + * absolutely necessary. Note that we can call objUse() multiple times, it + * handles that. + */ + CHKiRet(objUse(parser, CORE_COMPONENT)); + /* here, we create a new parser object */ + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parse", &pNew->mod.pm.parse)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"GetParserName", &GetParserName)); + CHKiRet(GetParserName(&pParserName)); + CHKiRet(parser.Construct(&pParser)); + + /* check some features */ + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticSanitazion); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoSanitazion(pParser, TRUE)); + } + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticPRIParsing); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoPRIParsing(pParser, TRUE)); + } + + CHKiRet(parser.SetName(pParser, pParserName)); + CHKiRet(parser.SetModPtr(pParser, pNew)); + CHKiRet(parser.ConstructFinalize(pParser)); + break; } pNew->pszName = (uchar*) strdup((char*)name); /* we do not care if strdup() fails, we can accept that */ @@ -460,14 +551,42 @@ static void modPrintList(void) case eMOD_LIB: dbgprintf("library"); break; + case eMOD_PARSER: + dbgprintf("parser"); + break; } dbgprintf(" module.\n"); dbgprintf("Entry points:\n"); dbgprintf("\tqueryEtryPt: 0x%lx\n", (unsigned long) pMod->modQueryEtryPt); - dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); - dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); dbgprintf("\tdbgPrintInstInfo: 0x%lx\n", (unsigned long) pMod->dbgPrintInstInfo); dbgprintf("\tfreeInstance: 0x%lx\n", (unsigned long) pMod->freeInstance); + switch(pMod->eType) { + case eMOD_OUT: + dbgprintf("Output Module Entry Points:\n"); + dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); + dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); + dbgprintf("\ttryResume: 0x%lx\n", (unsigned long) pMod->tryResume); + dbgprintf("\tdoHUP: 0x%lx\n", (unsigned long) pMod->doHUP); + dbgprintf("\tBeginTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.beginTransaction == dummyBeginTransaction) ? + 0 : pMod->mod.om.beginTransaction)); + dbgprintf("\tEndTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.endTransaction == dummyEndTransaction) ? + 0 : pMod->mod.om.endTransaction)); + break; + case eMOD_IN: + dbgprintf("Input Module Entry Points\n"); + dbgprintf("\trunInput: 0x%lx\n", (unsigned long) pMod->mod.im.runInput); + dbgprintf("\twillRun: 0x%lx\n", (unsigned long) pMod->mod.im.willRun); + dbgprintf("\tafterRun: 0x%lx\n", (unsigned long) pMod->mod.im.afterRun); + break; + case eMOD_LIB: + break; + case eMOD_PARSER: + dbgprintf("Parser Module Entry Points\n"); + dbgprintf("\tparse: 0x%lx\n", (unsigned long) pMod->mod.pm.parse); + break; + } dbgprintf("\n"); pMod = GetNxt(pMod); /* done, go next */ } @@ -806,6 +925,7 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); /* We have a problem in our reference counting, which leads to this function * being called too early. This usually is no problem, but if we destroy * the mutex object, we get into trouble. So rather than finding the root cause, diff --git a/runtime/modules.h b/runtime/modules.h index 372529ee..62f86ded 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -12,7 +12,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -51,9 +51,10 @@ #define CURR_MOD_IF_VERSION 5 typedef enum eModType_ { - eMOD_IN, /* input module */ - eMOD_OUT, /* output module */ - eMOD_LIB /* library module - this module provides one or many interfaces */ + eMOD_IN = 0, /* input module */ + eMOD_OUT = 1, /* output module */ + eMOD_LIB = 2, /* library module */ + eMOD_PARSER = 3 /* parser module */ } eModType_t; @@ -73,7 +74,7 @@ typedef enum eModLinkType_ { eMOD_LINK_ALL /* special: all linkage types, e.g. for unload */ } eModLinkType_t; -typedef struct modInfo_s { +struct modInfo_s { struct modInfo_s *pPrev; /* support for creating a double linked module list */ struct modInfo_s *pNext; /* support for creating a linked module list */ int iIFVers; /* Interface version of module */ @@ -106,15 +107,21 @@ typedef struct modInfo_s { rsRetVal (*runInput)(thrdInfo_t*); /* function to gather input and submit to queue */ rsRetVal (*willRun)(void); /* function to gather input and submit to queue */ rsRetVal (*afterRun)(thrdInfo_t*); /* function to gather input and submit to queue */ + int bCanRun; /* cached value of whether willRun() succeeded */ } im; struct {/* data for output modules */ /* below: perform the configured action */ + rsRetVal (*beginTransaction)(void*); rsRetVal (*doAction)(uchar**, unsigned, void*); + rsRetVal (*endTransaction)(void*); rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); } om; struct { /* data for library modules */ - } fm; + } lm; + struct { /* data for parser modules */ + rsRetVal (*parse)(msg_t*); + } pm; } mod; void *pModHdlr; /* handler to the dynamic library holding the module */ # ifdef DEBUG @@ -123,7 +130,7 @@ typedef struct modInfo_s { */ modUsr_t *pModUsrRoot; # endif -} modInfo_t; +}; /* interfaces */ BEGINinterface(module) /* name must also be changed in ENDinterface macro! */ @@ -149,6 +156,5 @@ extern uchar *pModDir; /* read-only after startup */ #endif /* #ifndef MODULES_H_INCLUDED */ -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/msg.c b/runtime/msg.c index dbc3c779..629c6f24 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -35,6 +35,11 @@ #include <string.h> #include <assert.h> #include <ctype.h> +#include <sys/socket.h> +#include <netdb.h> +#if HAVE_MALLOC_H +# include <malloc.h> +#endif #include "rsyslog.h" #include "srUtils.h" #include "stringbuf.h" @@ -46,6 +51,9 @@ #include "regexp.h" #include "atomic.h" #include "unicode-helper.h" +#include "ruleset.h" +#include "prop.h" +#include "net.h" /* static data */ DEFobjStaticHelpers @@ -53,57 +61,470 @@ DEFobjCurrIf(var) DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) +DEFobjCurrIf(prop) +DEFobjCurrIf(net) + +static struct { + uchar *pszName; + short lenName; +} syslog_pri_names[192] = { + { UCHAR_CONSTANT("0"), 3}, + { UCHAR_CONSTANT("1"), 3}, + { UCHAR_CONSTANT("2"), 3}, + { UCHAR_CONSTANT("3"), 3}, + { UCHAR_CONSTANT("4"), 3}, + { UCHAR_CONSTANT("5"), 3}, + { UCHAR_CONSTANT("6"), 3}, + { UCHAR_CONSTANT("7"), 3}, + { UCHAR_CONSTANT("8"), 3}, + { UCHAR_CONSTANT("9"), 3}, + { UCHAR_CONSTANT("10"), 4}, + { UCHAR_CONSTANT("11"), 4}, + { UCHAR_CONSTANT("12"), 4}, + { UCHAR_CONSTANT("13"), 4}, + { UCHAR_CONSTANT("14"), 4}, + { UCHAR_CONSTANT("15"), 4}, + { UCHAR_CONSTANT("16"), 4}, + { UCHAR_CONSTANT("17"), 4}, + { UCHAR_CONSTANT("18"), 4}, + { UCHAR_CONSTANT("19"), 4}, + { UCHAR_CONSTANT("20"), 4}, + { UCHAR_CONSTANT("21"), 4}, + { UCHAR_CONSTANT("22"), 4}, + { UCHAR_CONSTANT("23"), 4}, + { UCHAR_CONSTANT("24"), 4}, + { UCHAR_CONSTANT("25"), 4}, + { UCHAR_CONSTANT("26"), 4}, + { UCHAR_CONSTANT("27"), 4}, + { UCHAR_CONSTANT("28"), 4}, + { UCHAR_CONSTANT("29"), 4}, + { UCHAR_CONSTANT("30"), 4}, + { UCHAR_CONSTANT("31"), 4}, + { UCHAR_CONSTANT("32"), 4}, + { UCHAR_CONSTANT("33"), 4}, + { UCHAR_CONSTANT("34"), 4}, + { UCHAR_CONSTANT("35"), 4}, + { UCHAR_CONSTANT("36"), 4}, + { UCHAR_CONSTANT("37"), 4}, + { UCHAR_CONSTANT("38"), 4}, + { UCHAR_CONSTANT("39"), 4}, + { UCHAR_CONSTANT("40"), 4}, + { UCHAR_CONSTANT("41"), 4}, + { UCHAR_CONSTANT("42"), 4}, + { UCHAR_CONSTANT("43"), 4}, + { UCHAR_CONSTANT("44"), 4}, + { UCHAR_CONSTANT("45"), 4}, + { UCHAR_CONSTANT("46"), 4}, + { UCHAR_CONSTANT("47"), 4}, + { UCHAR_CONSTANT("48"), 4}, + { UCHAR_CONSTANT("49"), 4}, + { UCHAR_CONSTANT("50"), 4}, + { UCHAR_CONSTANT("51"), 4}, + { UCHAR_CONSTANT("52"), 4}, + { UCHAR_CONSTANT("53"), 4}, + { UCHAR_CONSTANT("54"), 4}, + { UCHAR_CONSTANT("55"), 4}, + { UCHAR_CONSTANT("56"), 4}, + { UCHAR_CONSTANT("57"), 4}, + { UCHAR_CONSTANT("58"), 4}, + { UCHAR_CONSTANT("59"), 4}, + { UCHAR_CONSTANT("60"), 4}, + { UCHAR_CONSTANT("61"), 4}, + { UCHAR_CONSTANT("62"), 4}, + { UCHAR_CONSTANT("63"), 4}, + { UCHAR_CONSTANT("64"), 4}, + { UCHAR_CONSTANT("65"), 4}, + { UCHAR_CONSTANT("66"), 4}, + { UCHAR_CONSTANT("67"), 4}, + { UCHAR_CONSTANT("68"), 4}, + { UCHAR_CONSTANT("69"), 4}, + { UCHAR_CONSTANT("70"), 4}, + { UCHAR_CONSTANT("71"), 4}, + { UCHAR_CONSTANT("72"), 4}, + { UCHAR_CONSTANT("73"), 4}, + { UCHAR_CONSTANT("74"), 4}, + { UCHAR_CONSTANT("75"), 4}, + { UCHAR_CONSTANT("76"), 4}, + { UCHAR_CONSTANT("77"), 4}, + { UCHAR_CONSTANT("78"), 4}, + { UCHAR_CONSTANT("79"), 4}, + { UCHAR_CONSTANT("80"), 4}, + { UCHAR_CONSTANT("81"), 4}, + { UCHAR_CONSTANT("82"), 4}, + { UCHAR_CONSTANT("83"), 4}, + { UCHAR_CONSTANT("84"), 4}, + { UCHAR_CONSTANT("85"), 4}, + { UCHAR_CONSTANT("86"), 4}, + { UCHAR_CONSTANT("87"), 4}, + { UCHAR_CONSTANT("88"), 4}, + { UCHAR_CONSTANT("89"), 4}, + { UCHAR_CONSTANT("90"), 4}, + { UCHAR_CONSTANT("91"), 4}, + { UCHAR_CONSTANT("92"), 4}, + { UCHAR_CONSTANT("93"), 4}, + { UCHAR_CONSTANT("94"), 4}, + { UCHAR_CONSTANT("95"), 4}, + { UCHAR_CONSTANT("96"), 4}, + { UCHAR_CONSTANT("97"), 4}, + { UCHAR_CONSTANT("98"), 4}, + { UCHAR_CONSTANT("99"), 4}, + { UCHAR_CONSTANT("100"), 5}, + { UCHAR_CONSTANT("101"), 5}, + { UCHAR_CONSTANT("102"), 5}, + { UCHAR_CONSTANT("103"), 5}, + { UCHAR_CONSTANT("104"), 5}, + { UCHAR_CONSTANT("105"), 5}, + { UCHAR_CONSTANT("106"), 5}, + { UCHAR_CONSTANT("107"), 5}, + { UCHAR_CONSTANT("108"), 5}, + { UCHAR_CONSTANT("109"), 5}, + { UCHAR_CONSTANT("110"), 5}, + { UCHAR_CONSTANT("111"), 5}, + { UCHAR_CONSTANT("112"), 5}, + { UCHAR_CONSTANT("113"), 5}, + { UCHAR_CONSTANT("114"), 5}, + { UCHAR_CONSTANT("115"), 5}, + { UCHAR_CONSTANT("116"), 5}, + { UCHAR_CONSTANT("117"), 5}, + { UCHAR_CONSTANT("118"), 5}, + { UCHAR_CONSTANT("119"), 5}, + { UCHAR_CONSTANT("120"), 5}, + { UCHAR_CONSTANT("121"), 5}, + { UCHAR_CONSTANT("122"), 5}, + { UCHAR_CONSTANT("123"), 5}, + { UCHAR_CONSTANT("124"), 5}, + { UCHAR_CONSTANT("125"), 5}, + { UCHAR_CONSTANT("126"), 5}, + { UCHAR_CONSTANT("127"), 5}, + { UCHAR_CONSTANT("128"), 5}, + { UCHAR_CONSTANT("129"), 5}, + { UCHAR_CONSTANT("130"), 5}, + { UCHAR_CONSTANT("131"), 5}, + { UCHAR_CONSTANT("132"), 5}, + { UCHAR_CONSTANT("133"), 5}, + { UCHAR_CONSTANT("134"), 5}, + { UCHAR_CONSTANT("135"), 5}, + { UCHAR_CONSTANT("136"), 5}, + { UCHAR_CONSTANT("137"), 5}, + { UCHAR_CONSTANT("138"), 5}, + { UCHAR_CONSTANT("139"), 5}, + { UCHAR_CONSTANT("140"), 5}, + { UCHAR_CONSTANT("141"), 5}, + { UCHAR_CONSTANT("142"), 5}, + { UCHAR_CONSTANT("143"), 5}, + { UCHAR_CONSTANT("144"), 5}, + { UCHAR_CONSTANT("145"), 5}, + { UCHAR_CONSTANT("146"), 5}, + { UCHAR_CONSTANT("147"), 5}, + { UCHAR_CONSTANT("148"), 5}, + { UCHAR_CONSTANT("149"), 5}, + { UCHAR_CONSTANT("150"), 5}, + { UCHAR_CONSTANT("151"), 5}, + { UCHAR_CONSTANT("152"), 5}, + { UCHAR_CONSTANT("153"), 5}, + { UCHAR_CONSTANT("154"), 5}, + { UCHAR_CONSTANT("155"), 5}, + { UCHAR_CONSTANT("156"), 5}, + { UCHAR_CONSTANT("157"), 5}, + { UCHAR_CONSTANT("158"), 5}, + { UCHAR_CONSTANT("159"), 5}, + { UCHAR_CONSTANT("160"), 5}, + { UCHAR_CONSTANT("161"), 5}, + { UCHAR_CONSTANT("162"), 5}, + { UCHAR_CONSTANT("163"), 5}, + { UCHAR_CONSTANT("164"), 5}, + { UCHAR_CONSTANT("165"), 5}, + { UCHAR_CONSTANT("166"), 5}, + { UCHAR_CONSTANT("167"), 5}, + { UCHAR_CONSTANT("168"), 5}, + { UCHAR_CONSTANT("169"), 5}, + { UCHAR_CONSTANT("170"), 5}, + { UCHAR_CONSTANT("171"), 5}, + { UCHAR_CONSTANT("172"), 5}, + { UCHAR_CONSTANT("173"), 5}, + { UCHAR_CONSTANT("174"), 5}, + { UCHAR_CONSTANT("175"), 5}, + { UCHAR_CONSTANT("176"), 5}, + { UCHAR_CONSTANT("177"), 5}, + { UCHAR_CONSTANT("178"), 5}, + { UCHAR_CONSTANT("179"), 5}, + { UCHAR_CONSTANT("180"), 5}, + { UCHAR_CONSTANT("181"), 5}, + { UCHAR_CONSTANT("182"), 5}, + { UCHAR_CONSTANT("183"), 5}, + { UCHAR_CONSTANT("184"), 5}, + { UCHAR_CONSTANT("185"), 5}, + { UCHAR_CONSTANT("186"), 5}, + { UCHAR_CONSTANT("187"), 5}, + { UCHAR_CONSTANT("188"), 5}, + { UCHAR_CONSTANT("189"), 5}, + { UCHAR_CONSTANT("190"), 5}, + { UCHAR_CONSTANT("191"), 5} + }; + +/*syslog facility names (as of RFC5424) */ +static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", "syslog", "lpr", + "news", "uucp", "cron", "authpriv", "ftp", "ntp", "audit", + "alert", "clock", "local0", "local1", "local2", "local3", + "local4", "local5", "local6", "local7" }; + +/* table of severity names (in numerical order)*/ +static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" }; + +/* numerical values as string - this is the most efficient approach to convert severity + * and facility values to a numerical string... -- rgerhars, 2009-06-17 + */ -static syslogCODE rs_prioritynames[] = - { - { "alert", LOG_ALERT }, - { "crit", LOG_CRIT }, - { "debug", LOG_DEBUG }, - { "emerg", LOG_EMERG }, - { "err", LOG_ERR }, - { "error", LOG_ERR }, /* DEPRECATED */ - { "info", LOG_INFO }, - { "none", INTERNAL_NOPRI }, /* INTERNAL */ - { "notice", LOG_NOTICE }, - { "panic", LOG_EMERG }, /* DEPRECATED */ - { "warn", LOG_WARNING }, /* DEPRECATED */ - { "warning", LOG_WARNING }, - { NULL, -1 } - }; - -#ifndef LOG_AUTHPRIV -# define LOG_AUTHPRIV LOG_AUTH -#endif -static syslogCODE rs_facilitynames[] = - { - { "auth", LOG_AUTH }, - { "authpriv", LOG_AUTHPRIV }, - { "cron", LOG_CRON }, - { "daemon", LOG_DAEMON }, -#if defined(LOG_FTP) - {"ftp", LOG_FTP}, -#endif - { "kern", LOG_KERN }, - { "lpr", LOG_LPR }, - { "mail", LOG_MAIL }, - { "news", LOG_NEWS }, - { "security", LOG_AUTH }, /* DEPRECATED */ - { "syslog", LOG_SYSLOG }, - { "user", LOG_USER }, - { "uucp", LOG_UUCP }, - { "local0", LOG_LOCAL0 }, - { "local1", LOG_LOCAL1 }, - { "local2", LOG_LOCAL2 }, - { "local3", LOG_LOCAL3 }, - { "local4", LOG_LOCAL4 }, - { "local5", LOG_LOCAL5 }, - { "local6", LOG_LOCAL6 }, - { "local7", LOG_LOCAL7 }, - { NULL, -1 } - }; +static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", + "15", "16", "17", "18", "19", "20", "21", "22", "23" }; /* some forward declarations */ -static int getAPPNAMELen(msg_t *pM); +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); + + +static inline int getProtocolVersion(msg_t *pM) +{ + return(pM->iProtocolVersion); +} + + +/* do a DNS reverse resolution, if not already done, reflect status + * rgerhards, 2009-11-16 + */ +static inline rsRetVal +resolveDNS(msg_t *pMsg) { + rsRetVal localRet; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + DEFiRet; + + CHKiRet(objUse(net, CORE_COMPONENT)); + if(pMsg->msgFlags & NEEDS_DNSRESOL) { + localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); + if(localRet == RS_RET_OK) { + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + } + } +finalize_it: + if(iRet != RS_RET_OK) { + /* best we can do: remove property */ + MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); + prop.Destruct(&propFromHost); + } + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; +} + + +static inline void +getInputName(msg_t *pM, uchar **ppsz, int *plen) +{ + BEGINfunc + if(pM == NULL || pM->pInputName == NULL) { + *ppsz = UCHAR_CONSTANT(""); + *plen = 0; + } else { + prop.GetString(pM->pInputName, ppsz, plen); + } + ENDfunc +} + + +static inline uchar* +getRcvFromIP(msg_t *pM) +{ + uchar *psz; + int len; + BEGINfunc + if(pM == NULL) { + psz = UCHAR_CONSTANT(""); + } else { + resolveDNS(pM); /* make sure we have a resolved entry */ + if(pM->pRcvFromIP == NULL) + psz = UCHAR_CONSTANT(""); + else + prop.GetString(pM->pRcvFromIP, &psz, &len); + } + ENDfunc + return psz; +} + + + +/* map a property name (string) to a property ID */ +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) +{ + uchar *pName; + DEFiRet; + + assert(pCSPropName != NULL); + assert(pPropID != NULL); + pName = rsCStrGetSzStrNoNULL(pCSPropName); + + /* sometimes there are aliases to the original MonitoWare + * property names. These come after || in the ifs below. */ + if(!strcmp((char*) pName, "msg")) { + *pPropID = PROP_MSG; + } else if(!strcmp((char*) pName, "timestamp") + || !strcmp((char*) pName, "timereported")) { + *pPropID = PROP_TIMESTAMP; + } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { + *pPropID = PROP_HOSTNAME; + } else if(!strcmp((char*) pName, "syslogtag")) { + *pPropID = PROP_SYSLOGTAG; + } else if(!strcmp((char*) pName, "rawmsg")) { + *pPropID = PROP_RAWMSG; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(!strcmp((char*) pName, "uxtradmsg")) { + pRes = getUxTradMsg(pMsg); + */ + } else if(!strcmp((char*) pName, "inputname")) { + *pPropID = PROP_INPUTNAME; + } else if(!strcmp((char*) pName, "fromhost")) { + *pPropID = PROP_FROMHOST; + } else if(!strcmp((char*) pName, "fromhost-ip")) { + *pPropID = PROP_FROMHOST_IP; + } else if(!strcmp((char*) pName, "pri")) { + *pPropID = PROP_PRI; + } else if(!strcmp((char*) pName, "pri-text")) { + *pPropID = PROP_PRI_TEXT; + } else if(!strcmp((char*) pName, "iut")) { + *pPropID = PROP_IUT; + } else if(!strcmp((char*) pName, "syslogfacility")) { + *pPropID = PROP_SYSLOGFACILITY; + } else if(!strcmp((char*) pName, "syslogfacility-text")) { + *pPropID = PROP_SYSLOGFACILITY_TEXT; + } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { + *pPropID = PROP_SYSLOGSEVERITY; + } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { + *pPropID = PROP_SYSLOGSEVERITY_TEXT; + } else if(!strcmp((char*) pName, "timegenerated")) { + *pPropID = PROP_TIMEGENERATED; + } else if(!strcmp((char*) pName, "programname")) { + *pPropID = PROP_PROGRAMNAME; + } else if(!strcmp((char*) pName, "protocol-version")) { + *pPropID = PROP_PROTOCOL_VERSION; + } else if(!strcmp((char*) pName, "structured-data")) { + *pPropID = PROP_STRUCTURED_DATA; + } else if(!strcmp((char*) pName, "app-name")) { + *pPropID = PROP_APP_NAME; + } else if(!strcmp((char*) pName, "procid")) { + *pPropID = PROP_PROCID; + } else if(!strcmp((char*) pName, "msgid")) { + *pPropID = PROP_MSGID; + /* here start system properties (those, that do not relate to the message itself */ + } else if(!strcmp((char*) pName, "$now")) { + *pPropID = PROP_SYS_NOW; + } else if(!strcmp((char*) pName, "$year")) { + *pPropID = PROP_SYS_YEAR; + } else if(!strcmp((char*) pName, "$month")) { + *pPropID = PROP_SYS_MONTH; + } else if(!strcmp((char*) pName, "$day")) { + *pPropID = PROP_SYS_DAY; + } else if(!strcmp((char*) pName, "$hour")) { + *pPropID = PROP_SYS_HOUR; + } else if(!strcmp((char*) pName, "$hhour")) { + *pPropID = PROP_SYS_HHOUR; + } else if(!strcmp((char*) pName, "$qhour")) { + *pPropID = PROP_SYS_QHOUR; + } else if(!strcmp((char*) pName, "$minute")) { + *pPropID = PROP_SYS_MINUTE; + } else if(!strcmp((char*) pName, "$myhostname")) { + *pPropID = PROP_SYS_MYHOSTNAME; + } else { + *pPropID = PROP_INVALID; + iRet = RS_RET_VAR_NOT_FOUND; + } + + RETiRet; +} + + +/* map a property ID to a name string (useful for displaying) */ +uchar *propIDToName(propid_t propID) +{ + switch(propID) { + case PROP_MSG: + return UCHAR_CONSTANT("msg"); + case PROP_TIMESTAMP: + return UCHAR_CONSTANT("timestamp"); + case PROP_HOSTNAME: + return UCHAR_CONSTANT("hostname"); + case PROP_SYSLOGTAG: + return UCHAR_CONSTANT("syslogtag"); + case PROP_RAWMSG: + return UCHAR_CONSTANT("rawmsg"); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + case PROP_UXTRADMSG: + pRes = getUxTradMsg(pMsg); + break; + */ + case PROP_INPUTNAME: + return UCHAR_CONSTANT("inputname"); + case PROP_FROMHOST: + return UCHAR_CONSTANT("fromhost"); + case PROP_FROMHOST_IP: + return UCHAR_CONSTANT("fromhost-ip"); + case PROP_PRI: + return UCHAR_CONSTANT("pri"); + case PROP_PRI_TEXT: + return UCHAR_CONSTANT("pri-text"); + case PROP_IUT: + return UCHAR_CONSTANT("iut"); + case PROP_SYSLOGFACILITY: + return UCHAR_CONSTANT("syslogfacility"); + case PROP_SYSLOGFACILITY_TEXT: + return UCHAR_CONSTANT("syslogfacility-text"); + case PROP_SYSLOGSEVERITY: + return UCHAR_CONSTANT("syslogseverity"); + case PROP_SYSLOGSEVERITY_TEXT: + return UCHAR_CONSTANT("syslogseverity-text"); + case PROP_TIMEGENERATED: + return UCHAR_CONSTANT("timegenerated"); + case PROP_PROGRAMNAME: + return UCHAR_CONSTANT("programname"); + case PROP_PROTOCOL_VERSION: + return UCHAR_CONSTANT("protocol-version"); + case PROP_STRUCTURED_DATA: + return UCHAR_CONSTANT("structured-data"); + case PROP_APP_NAME: + return UCHAR_CONSTANT("app-name"); + case PROP_PROCID: + return UCHAR_CONSTANT("procid"); + case PROP_MSGID: + return UCHAR_CONSTANT("msgid"); + case PROP_SYS_NOW: + return UCHAR_CONSTANT("$NOW"); + case PROP_SYS_YEAR: + return UCHAR_CONSTANT("$YEAR"); + case PROP_SYS_MONTH: + return UCHAR_CONSTANT("$MONTH"); + case PROP_SYS_DAY: + return UCHAR_CONSTANT("$DAY"); + case PROP_SYS_HOUR: + return UCHAR_CONSTANT("$HOUR"); + case PROP_SYS_HHOUR: + return UCHAR_CONSTANT("$HHOUR"); + case PROP_SYS_QHOUR: + return UCHAR_CONSTANT("$QHOUR"); + case PROP_SYS_MINUTE: + return UCHAR_CONSTANT("$MINUTE"); + case PROP_SYS_MYHOSTNAME: + return UCHAR_CONSTANT("$MYHOSTNAME"); + default: + return UCHAR_CONSTANT("*invalid property id*"); + } +} + /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we @@ -165,32 +586,9 @@ static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) */ static void MsgPrepareEnqueueLockingCase(msg_t *pThis) { - int iErr; BEGINfunc assert(pThis != NULL); - iErr = pthread_mutexattr_init(&pThis->mutAttr); - if(iErr != 0) { - dbgprintf("error initializing mutex attribute in %s:%d, trying to continue\n", - __FILE__, __LINE__); - } - iErr = pthread_mutexattr_settype(&pThis->mutAttr, PTHREAD_MUTEX_RECURSIVE); - if(iErr != 0) { - dbgprintf("ERROR setting mutex attribute to recursive in %s:%d, trying to continue " - "but we will probably either abort or hang soon\n", - __FILE__, __LINE__); - /* TODO: it makes very little sense to continue here, - * but it requires an iRet interface to gracefully shut - * down. We should do that over time. -- rgerhards, 2008-07-14 - */ - } - pthread_mutex_init(&pThis->mut, &pThis->mutAttr); - - /* we do no longer need the attribute. According to the - * POSIX spec, we can destroy it without affecting the - * initialized mutex (that used the attribute). - * rgerhards, 2008-07-14 - */ - pthread_mutexattr_destroy(&pThis->mutAttr); + pthread_mutex_init(&pThis->mut, NULL); pThis->bDoLock = 1; ENDfunc } @@ -255,6 +653,11 @@ rsRetVal MsgEnableThreadSafety(void) * itself but rather uses a user-supplied value. This enables the caller * to do some tricks to save processing time (done, for example, in the * udp input). + * NOTE: this constructor does NOT call calloc(), as we have many bytes + * inside the structure which do not need to be cleared. bzero() will + * heavily thrash the cache, so we do the init manually (which also + * is the right thing to do with pointers, as they are not neccessarily + * a binary 0 on all machines [but today almost always...]). * rgerhards, 2008-10-06 */ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) @@ -263,14 +666,49 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) msg_t *pM; assert(ppThis != NULL); - if((pM = calloc(1, sizeof(msg_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pM = MALLOC(sizeof(msg_t))); + objConstructSetObjInfo(pM); /* intialize object helper entities */ - /* initialize members that are non-zero */ + /* initialize members in ORDER they appear in structure (think "cache line"!) */ + pM->flowCtlType = 0; + pM->bDoLock = 0; pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; - objConstructSetObjInfo(pM); + pM->offAfterPRI = 0; + pM->offMSG = -1; + pM->iProtocolVersion = 0; + pM->msgFlags = 0; + pM->iLenRawMsg = 0; + pM->iLenMSG = 0; + pM->iLenTAG = 0; + pM->iLenHOSTNAME = 0; + pM->pszRawMsg = NULL; + pM->pszHOSTNAME = NULL; + pM->pszRcvdAt3164 = NULL; + pM->pszRcvdAt3339 = NULL; + pM->pszRcvdAt_MySQL = NULL; + pM->pszRcvdAt_PgSQL = NULL; + pM->pszTIMESTAMP3164 = NULL; + pM->pszTIMESTAMP3339 = NULL; + pM->pszTIMESTAMP_MySQL = NULL; + pM->pszTIMESTAMP_PgSQL = NULL; + pM->pCSProgName = NULL; + pM->pCSStrucData = NULL; + pM->pCSAPPNAME = NULL; + pM->pCSPROCID = NULL; + pM->pCSMSGID = NULL; + pM->pInputName = NULL; + pM->pRcvFromIP = NULL; + pM->rcvFrom.pRcvFrom = NULL; + pM->pRuleset = NULL; + memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); + memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); + pM->TAG.pszTAG = NULL; + pM->pszTimestamp3164[0] = '\0'; + pM->pszTimestamp3339[0] = '\0'; + pM->pszTIMESTAMP_SecFrac[0] = '\0'; + pM->pszRcvdAt_SecFrac[0] = '\0'; /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ @@ -329,6 +767,21 @@ finalize_it: } +/* some free handlers for (slightly) complicated cases... All of them may be called + * with an empty element. + */ +static inline void freeTAG(msg_t *pThis) +{ + if(pThis->iLenTAG >= CONF_TAG_BUFSIZE) + free(pThis->TAG.pszTAG); +} +static inline void freeHOSTNAME(msg_t *pThis) +{ + if(pThis->iLenHOSTNAME >= CONF_HOSTNAME_BUFSIZE) + free(pThis->pszHOSTNAME); +} + + BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ int currRefCount; CODESTARTobjDestruct(msg) @@ -342,29 +795,26 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - free(pThis->pszUxTradMsg); - free(pThis->pszRawMsg); - free(pThis->pszTAG); - free(pThis->pszHOSTNAME); - free(pThis->pszInputName); - free(pThis->pszRcvFrom); - free(pThis->pszRcvFromIP); - free(pThis->pszMSG); - free(pThis->pszFacility); - free(pThis->pszFacilityStr); - free(pThis->pszSeverity); - free(pThis->pszSeverityStr); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); + freeTAG(pThis); + freeHOSTNAME(pThis); + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); + if((pThis->msgFlags & NEEDS_DNSRESOL) == 0) { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } else { + free(pThis->rcvFrom.pfrominet); + } + if(pThis->pRcvFromIP != NULL) + prop.Destruct(&pThis->pRcvFromIP); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); - free(pThis->pszRcvdAt_SecFrac); free(pThis->pszRcvdAt_MySQL); free(pThis->pszRcvdAt_PgSQL); - free(pThis->pszTIMESTAMP3164); - free(pThis->pszTIMESTAMP3339); - free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); - free(pThis->pszPRI); if(pThis->pCSProgName != NULL) rsCStrDestruct(&pThis->pCSProgName); if(pThis->pCSStrucData != NULL) @@ -379,8 +829,29 @@ CODESTARTobjDestruct(msg) MsgUnlock(pThis); # endif funcDeleteMutex(pThis); + /* now we need to do our own optimization. Testing has shown that at least the glibc + * malloc() subsystem returns memory to the OS far too late in our case. So we need + * to help it a bit, by calling malloc_trim(), which will tell the alloc subsystem + * to consolidate and return to the OS. We keep 128K for our use, as a safeguard + * to too-frequent reallocs. But more importantly, we call this hook only every + * 100,000 messages (which is an approximation, as we do not work with atomic + * operations on the counter. --- rgerhards, 2009-06-22. + */ +# if HAVE_MALLOC_TRIM + { /* standard C requires a new block for a new variable definition! + * To simplify matters, we use modulo arithmetic and live with the fact + * that we trim too often when the counter wraps. + */ + static unsigned iTrimCtr = 1; + if(ATOMIC_INC_AND_FETCH(iTrimCtr) % 100000 == 0) { + malloc_trim(128*1024); + } + } +# endif } else { +# ifndef HAVE_ATOMIC_BUILTINS MsgUnlock(pThis); +# endif pThis = NULL; /* tell framework not to destructing the object! */ } ENDobjDestruct(msg) @@ -421,6 +892,7 @@ ENDobjDestruct(msg) msg_t* MsgDup(msg_t* pOld) { msg_t* pNew; + rsRetVal localRet; assert(pOld != NULL); @@ -433,21 +905,63 @@ msg_t* MsgDup(msg_t* pOld) pNew->iRefCount = 1; pNew->iSeverity = pOld->iSeverity; pNew->iFacility = pOld->iFacility; - pNew->bParseHOSTNAME = pOld->bParseHOSTNAME; pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; - tmpCOPYSZ(Severity); - tmpCOPYSZ(SeverityStr); - tmpCOPYSZ(Facility); - tmpCOPYSZ(FacilityStr); - tmpCOPYSZ(PRI); - tmpCOPYSZ(RawMsg); - tmpCOPYSZ(MSG); - tmpCOPYSZ(UxTradMsg); - tmpCOPYSZ(TAG); - tmpCOPYSZ(HOSTNAME); - tmpCOPYSZ(RcvFrom); + pNew->offMSG = pOld->offMSG; + pNew->iLenRawMsg = pOld->iLenRawMsg; + pNew->iLenMSG = pOld->iLenMSG; + pNew->iLenTAG = pOld->iLenTAG; + pNew->iLenHOSTNAME = pOld->iLenHOSTNAME; + if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) { + localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet); + if(localRet != RS_RET_OK) { + /* if something fails, we accept loss of this property, it is + * better than losing the whole message. + */ + pNew->msgFlags &= ~NEEDS_DNSRESOL; + } + } else { + if(pOld->rcvFrom.pRcvFrom != NULL) { + pNew->rcvFrom.pRcvFrom = pOld->rcvFrom.pRcvFrom; + prop.AddRef(pNew->rcvFrom.pRcvFrom); + } + } + if(pOld->pRcvFromIP != NULL) { + pNew->pRcvFromIP = pOld->pRcvFromIP; + prop.AddRef(pNew->pRcvFromIP); + } + if(pOld->pInputName != NULL) { + pNew->pInputName = pOld->pInputName; + prop.AddRef(pNew->pInputName); + } + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + pNew->offAfterPRI = pOld->offAfterPRI; + */ + if(pOld->iLenTAG > 0) { + if(pOld->iLenTAG < CONF_TAG_BUFSIZE) { + memcpy(pNew->TAG.szBuf, pOld->TAG.szBuf, pOld->iLenTAG); + } else { + if((pNew->TAG.pszTAG = srUtilStrDup(pOld->TAG.pszTAG, pOld->iLenTAG)) == NULL) { + msgDestruct(&pNew); + return NULL; + } + pNew->iLenTAG = pOld->iLenTAG; + } + } + if(pOld->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + memcpy(pNew->szRawMsg, pOld->szRawMsg, pOld->iLenRawMsg + 1); + pNew->pszRawMsg = pNew->szRawMsg; + } else { + tmpCOPYSZ(RawMsg); + } + if(pOld->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { + memcpy(pNew->szHOSTNAME, pOld->szHOSTNAME, pOld->iLenHOSTNAME + 1); + pNew->pszHOSTNAME = pNew->szHOSTNAME; + } else { + tmpCOPYSZ(HOSTNAME); + } tmpCOPYCSTR(ProgName); tmpCOPYCSTR(StrucData); @@ -474,17 +988,20 @@ msg_t* MsgDup(msg_t* pOld) * We do not serialize the cache properties. We re-create them when needed. * This saves us a lot of memory. Performance is no concern, as serializing * is a so slow operation that recration of the caches does not count. Also, - * we do not serialize bParseHOSTNAME, as this is only a helper variable + * we do not serialize --currently none--, as this is only a helper variable * during msg construction - and never again used later. * rgerhards, 2008-01-03 */ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) { + uchar *psz; + int len; DEFiRet; assert(pThis != NULL); assert(pStrm != NULL); + /* then serialize elements */ CHKiRet(obj.BeginSerialize(pStrm, (obj_t*) pThis)); objSerializeSCALAR(pStrm, iProtocolVersion, SHORT); objSerializeSCALAR(pStrm, iSeverity, SHORT); @@ -493,21 +1010,33 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + objSerializeSCALAR(pStrm, offsAfterPRI, SHORT); + */ + + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszTAG"), PROPTYPE_PSZ, (void*) + ((pThis->iLenTAG < CONF_TAG_BUFSIZE) ? pThis->TAG.szBuf : pThis->TAG.pszTAG))); objSerializePTR(pStrm, pszRawMsg, PSZ); - objSerializePTR(pStrm, pszMSG, PSZ); - objSerializePTR(pStrm, pszUxTradMsg, PSZ); - objSerializePTR(pStrm, pszTAG, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); - objSerializePTR(pStrm, pszInputName, PSZ); - objSerializePTR(pStrm, pszRcvFrom, PSZ); - objSerializePTR(pStrm, pszRcvFromIP, PSZ); + getInputName(pThis, &psz, &len); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszInputName"), PROPTYPE_PSZ, (void*) psz)); + psz = getRcvFrom(pThis); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFrom"), PROPTYPE_PSZ, (void*) psz)); + psz = getRcvFromIP(pThis); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFromIP"), PROPTYPE_PSZ, (void*) psz)); objSerializePTR(pStrm, pCSStrucData, CSTR); objSerializePTR(pStrm, pCSAPPNAME, CSTR); objSerializePTR(pStrm, pCSPROCID, CSTR); objSerializePTR(pStrm, pCSMSGID, CSTR); + /* offset must be serialized after pszRawMsg, because we need that to obtain the correct + * MSG size. + */ + objSerializeSCALAR(pStrm, offMSG, SHORT); + CHKiRet(obj.EndSerialize(pStrm)); finalize_it: @@ -546,22 +1075,27 @@ msg_t *MsgAddRef(msg_t *pM) * can obtain a PROCID. Take in mind that not every legacy syslog message * actually has a PROCID. * rgerhards, 2005-11-24 + * THIS MUST be called with the message lock locked. */ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) { register int i; + uchar *pszTag; DEFiRet; assert(pM != NULL); + if(pM->pCSPROCID != NULL) return RS_RET_OK; /* we are already done ;) */ if(getProtocolVersion(pM) != 0) return RS_RET_OK; /* we can only emulate if we have legacy format */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); + /* find first '['... */ i = 0; - while((i < pM->iLenTAG) && (pM->pszTAG[i] != '[')) + while((i < pM->iLenTAG) && (pszTag[i] != '[')) ++i; if(!(i < pM->iLenTAG)) return RS_RET_OK; /* no [, so can not emulate... */ @@ -569,10 +1103,9 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) ++i; /* skip '[' */ /* now obtain the PROCID string... */ - CHKiRet(rsCStrConstruct(&pM->pCSPROCID)); - rsCStrSetAllocIncrement(pM->pCSPROCID, 16); - while((i < pM->iLenTAG) && (pM->pszTAG[i] != ']')) { - CHKiRet(rsCStrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); + CHKiRet(cstrConstruct(&pM->pCSPROCID)); + while((i < pM->iLenTAG) && (pszTag[i] != ']')) { + CHKiRet(cstrAppendChar(pM->pCSPROCID, pszTag[i])); ++i; } @@ -582,12 +1115,12 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) * the buffer and simply return. Note that this is NOT an error * case! */ - rsCStrDestruct(&pM->pCSPROCID); + cstrDestruct(&pM->pCSPROCID); FINALIZE; } /* OK, finaally we could obtain a PROCID. So let's use it ;) */ - CHKiRet(rsCStrFinish(pM->pCSPROCID)); + CHKiRet(cstrFinalize(pM->pCSPROCID)); finalize_it: RETiRet; @@ -607,57 +1140,35 @@ finalize_it: * The program name is not parsed by default, because it is infrequently-used. * If it is needed, this function should be called first. It checks if it is * already set and extracts it, if not. - * A message object must be provided, else a crash will occur. + * + * IMPORTANT: A locked message object must be provided, else a crash will occur. * rgerhards, 2005-10-19 */ static rsRetVal aquireProgramName(msg_t *pM) { - DEFiRet; register int i; + uchar *pszTag; + DEFiRet; assert(pM != NULL); if(pM->pCSProgName == NULL) { - /* ok, we do not yet have it. So let's parse the TAG - * to obtain it. - */ - CHKiRet(rsCStrConstruct(&pM->pCSProgName)); - rsCStrSetAllocIncrement(pM->pCSProgName, 33); + /* ok, we do not yet have it. So let's parse the TAG to obtain it. */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); + CHKiRet(cstrConstruct(&pM->pCSProgName)); for( i = 0 - ; (i < pM->iLenTAG) && isprint((int) pM->pszTAG[i]) - && (pM->pszTAG[i] != '\0') && (pM->pszTAG[i] != ':') - && (pM->pszTAG[i] != '[') && (pM->pszTAG[i] != '/') + ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) + && (pszTag[i] != '\0') && (pszTag[i] != ':') + && (pszTag[i] != '[') && (pszTag[i] != '/') ; ++i) { - CHKiRet(rsCStrAppendChar(pM->pCSProgName, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSProgName, pszTag[i])); } - CHKiRet(rsCStrFinish(pM->pCSProgName)); + CHKiRet(cstrFinalize(pM->pCSProgName)); } finalize_it: RETiRet; } -/* This function moves the HOSTNAME inside the message object to the - * TAG. It is a specialised function used to handle the condition when - * a message without HOSTNAME is being processed. The missing HOSTNAME - * is only detected at a later stage, during TAG processing, so that - * we already had set the HOSTNAME property and now need to move it to - * the TAG. Of course, we could do this via a couple of get/set methods, - * but it is far more efficient to do it via this specialised method. - * This is especially important as this can be a very common case, e.g. - * when BSD syslog is acting as a sender. - * rgerhards, 2005-11-10. - */ -void moveHOSTNAMEtoTAG(msg_t *pM) -{ - assert(pM != NULL); - if(pM->pszTAG != NULL) - free(pM->pszTAG); - pM->pszTAG = pM->pszHOSTNAME; - pM->iLenTAG = pM->iLenHOSTNAME; - pM->pszHOSTNAME = NULL; - pM->iLenHOSTNAME = 0; -} - /* Access methods - dumb & easy, not a comment for each ;) */ void setProtocolVersion(msg_t *pM, int iNewVersion) @@ -670,12 +1181,6 @@ void setProtocolVersion(msg_t *pM, int iNewVersion) pM->iProtocolVersion = iNewVersion; } -int getProtocolVersion(msg_t *pM) -{ - assert(pM != NULL); - return(pM->iProtocolVersion); -} - /* note: string is taken from constant pool, do NOT free */ char *getProtocolVersionString(msg_t *pM) { @@ -683,84 +1188,82 @@ char *getProtocolVersionString(msg_t *pM) return(pM->iProtocolVersion ? "1" : "0"); } -int getMSGLen(msg_t *pM) + +static inline void +getRawMsg(msg_t *pM, uchar **pBuf, int *piLen) { - return((pM == NULL) ? 0 : pM->iLenMSG); + if(pM == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + if(pM->pszRawMsg == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + *pBuf = pM->pszRawMsg; + *piLen = pM->iLenRawMsg; + } + } } -char *getRawMsg(msg_t *pM) +/* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 +char *getUxTradMsg(msg_t *pM) { if(pM == NULL) return ""; else - if(pM->pszRawMsg == NULL) - return ""; - else - return (char*)pM->pszRawMsg; + return (char*)pM->pszRawMsg + pM->offAfterPRI; } +*/ -char *getUxTradMsg(msg_t *pM) + +int getMSGLen(msg_t *pM) { - if(pM == NULL) - return ""; - else - if(pM->pszUxTradMsg == NULL) - return ""; - else - return (char*)pM->pszUxTradMsg; + return((pM == NULL) ? 0 : pM->iLenMSG); } -char *getMSG(msg_t *pM) +uchar *getMSG(msg_t *pM) { + uchar *ret; if(pM == NULL) - return ""; - else - if(pM->pszMSG == NULL) - return ""; + ret = UCHAR_CONSTANT(""); + else { + if(pM->iLenMSG == 0) + ret = UCHAR_CONSTANT(""); else - return (char*)pM->pszMSG; + ret = pM->pszRawMsg + pM->offMSG; + } + return ret; } -/* Get PRI value in text form */ -char *getPRI(msg_t *pM) +/* Get PRI value as integer */ +static int getPRIi(msg_t *pM) { - int pri; - BEGINfunc - - if(pM == NULL) - return ""; - - MsgLock(pM); - if(pM->pszPRI == NULL) { - /* OK, we need to construct it... we use a 5 byte buffer - as of - * RFC 3164, it can't be longer. Should it still be, snprintf will truncate... - * Note that we do not use the LOG_MAKEPRI macro. This macro - * is a simple add of the two values under FreeBSD 7. So we implement - * the logic in our own code. This is a change from a bug - * report. -- rgerhards, 2008-07-14 - */ - pri = pM->iFacility * 8 + pM->iSeverity; - if((pM->pszPRI = malloc(5)) == NULL) return ""; - pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); - } - MsgUnlock(pM); - - ENDfunc - return (char*)pM->pszPRI; + return (pM->iFacility << 3) + (pM->iSeverity); } -/* Get PRI value as integer */ -int getPRIi(msg_t *pM) +/* Get PRI value in text form + */ +static inline char *getPRI(msg_t *pM) { - assert(pM != NULL); - return (pM->iFacility << 3) + (pM->iSeverity); + /* PRI is a number in the range 0..191. Thus, we use a simple lookup table to obtain the + * string value. It looks a bit clumpsy here in code ;) + */ + int iPRI; + + if(pM == NULL) + return ""; + + iPRI = getPRIi(pM); + return (iPRI > 191) ? "invld" : (char*)syslog_pri_names[iPRI].pszName; } -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -768,77 +1271,60 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) switch(eFmt) { case tplFmtDefault: + case tplFmtRFC3164Date: MsgLock(pM); if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } - datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); + pM->pszTIMESTAMP3164 = pM->pszTimestamp3164; + datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164); } MsgUnlock(pM); return(pM->pszTIMESTAMP3164); case tplFmtMySQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_MySQL == NULL) { - if((pM->pszTIMESTAMP_MySQL = malloc(15)) == NULL) { + if((pM->pszTIMESTAMP_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToMySQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_MySQL, 15); + datetime.formatTimestampToMySQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_MySQL); } MsgUnlock(pM); return(pM->pszTIMESTAMP_MySQL); case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_PgSQL == NULL) { - if((pM->pszTIMESTAMP_PgSQL = malloc(21)) == NULL) { + if((pM->pszTIMESTAMP_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToPgSQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_PgSQL, 21); + datetime.formatTimestampToPgSQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_PgSQL); } MsgUnlock(pM); return(pM->pszTIMESTAMP_PgSQL); - case tplFmtRFC3164Date: - MsgLock(pM); - if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } - datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); - } - MsgUnlock(pM); - return(pM->pszTIMESTAMP3164); case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszTIMESTAMP3339 == NULL) { - if((pM->pszTIMESTAMP3339 = malloc(33)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ - } - datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339, 33); + pM->pszTIMESTAMP3339 = pM->pszTimestamp3339; + datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339); } MsgUnlock(pM); return(pM->pszTIMESTAMP3339); case tplFmtSecFrac: - MsgLock(pM); - if(pM->pszTIMESTAMP_SecFrac == NULL) { - if((pM->pszTIMESTAMP_SecFrac = malloc(10)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ + if(pM->pszTIMESTAMP_SecFrac[0] == '\0') { + MsgLock(pM); + /* re-check, may have changed while we did not hold lock */ + if(pM->pszTIMESTAMP_SecFrac[0] == '\0') { + datetime.formatTimestampSecFrac(&pM->tTIMESTAMP, pM->pszTIMESTAMP_SecFrac); } - datetime.formatTimestampSecFrac(&pM->tTIMESTAMP, pM->pszTIMESTAMP_SecFrac, 10); + MsgUnlock(pM); } - MsgUnlock(pM); return(pM->pszTIMESTAMP_SecFrac); } ENDfunc return "INVALID eFmt OPTION!"; } -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -848,68 +1334,67 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtDefault: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164, 16); + datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164); } MsgUnlock(pM); return(pM->pszRcvdAt3164); case tplFmtMySQLDate: MsgLock(pM); if(pM->pszRcvdAt_MySQL == NULL) { - if((pM->pszRcvdAt_MySQL = malloc(15)) == NULL) { + if((pM->pszRcvdAt_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToMySQL(&pM->tRcvdAt, pM->pszRcvdAt_MySQL, 15); + datetime.formatTimestampToMySQL(&pM->tRcvdAt, pM->pszRcvdAt_MySQL); } MsgUnlock(pM); return(pM->pszRcvdAt_MySQL); case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszRcvdAt_PgSQL == NULL) { - if((pM->pszRcvdAt_PgSQL = malloc(21)) == NULL) { + if((pM->pszRcvdAt_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToPgSQL(&pM->tRcvdAt, pM->pszRcvdAt_PgSQL, 21); + datetime.formatTimestampToPgSQL(&pM->tRcvdAt, pM->pszRcvdAt_PgSQL); } MsgUnlock(pM); return(pM->pszRcvdAt_PgSQL); case tplFmtRFC3164Date: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164, 16); + datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164); } MsgUnlock(pM); return(pM->pszRcvdAt3164); case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszRcvdAt3339 == NULL) { - if((pM->pszRcvdAt3339 = malloc(33)) == NULL) { + if((pM->pszRcvdAt3339 = MALLOC(33)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestamp3339(&pM->tRcvdAt, pM->pszRcvdAt3339, 33); + datetime.formatTimestamp3339(&pM->tRcvdAt, pM->pszRcvdAt3339); } MsgUnlock(pM); return(pM->pszRcvdAt3339); case tplFmtSecFrac: - MsgLock(pM); - if(pM->pszRcvdAt_SecFrac == NULL) { - if((pM->pszRcvdAt_SecFrac = malloc(10)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ + if(pM->pszRcvdAt_SecFrac[0] == '\0') { + MsgLock(pM); + /* re-check, may have changed while we did not hold lock */ + if(pM->pszRcvdAt_SecFrac[0] == '\0') { + datetime.formatTimestampSecFrac(&pM->tRcvdAt, pM->pszRcvdAt_SecFrac); } - datetime.formatTimestampSecFrac(&pM->tRcvdAt, pM->pszRcvdAt_SecFrac, 10); + MsgUnlock(pM); } - MsgUnlock(pM); return(pM->pszRcvdAt_SecFrac); } ENDfunc @@ -917,103 +1402,69 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) } -char *getSeverity(msg_t *pM) +static inline char *getSeverity(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverity == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverity = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverity = - snprintf((char*)pM->pszSeverity, 2, "%d", pM->iSeverity); + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_number_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverity); + + return name; } -char *getSeverityStr(msg_t *pM) +static inline char *getSeverityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverityStr == NULL) { - for(c = rs_prioritynames, val = pM->iSeverity; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverityStr = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = - snprintf((char*)pM->pszSeverityStr, 2, "%d", pM->iSeverity); - } else { - if((pM->pszSeverityStr = (uchar*) strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = strlen((char*)name); - } + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_severity_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverityStr); + + return name; } -char *getFacility(msg_t *pM) +static inline char *getFacility(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacility == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacility = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacility = - snprintf((char*)pM->pszFacility, 12, "%d", pM->iFacility); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_number_names[pM->iFacility]; } - MsgUnlock(pM); - return((char*)pM->pszFacility); + + return name; } -char *getFacilityStr(msg_t *pM) +static inline char *getFacilityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacilityStr == NULL) { - for(c = rs_facilitynames, val = pM->iFacility << 3; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacilityStr = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = - snprintf((char*)pM->pszFacilityStr, 12, "%d", val >> 3); - } else { - if((pM->pszFacilityStr = (uchar*)strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = strlen((char*)name); - } - } - MsgUnlock(pM); - return((char*)pM->pszFacilityStr); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_fac_names[pM->iFacility]; + } + + return name; } @@ -1035,9 +1486,23 @@ MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl) RETiRet; } +/* set offset after which PRI in raw msg starts + * rgerhards, 2009-06-16 + */ +rsRetVal +MsgSetAfterPRIOffs(msg_t *pMsg, short offs) +{ + assert(pMsg != NULL); + pMsg->offAfterPRI = offs; + return RS_RET_OK; +} + /* rgerhards 2004-11-24: set APP-NAME in msg object - * TODO: revisit msg locking code! + * This is not locked, because it either is called during message + * construction (where we need no locking) or later as part of a function + * which already obtained the lock. So in general, this function here must + * only be called when it it safe to do so without it aquiring a lock. */ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME) { @@ -1046,7 +1511,6 @@ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME) if(pMsg->pCSAPPNAME == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSAPPNAME)); - rsCStrSetAllocIncrement(pMsg->pCSAPPNAME, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSAPPNAME, (uchar*) pszAPPNAME); @@ -1056,20 +1520,6 @@ finalize_it: } -static void tryEmulateAPPNAME(msg_t *pM); /* forward reference */ -/* rgerhards, 2005-11-24 - */ -char *getAPPNAME(msg_t *pM) -{ - assert(pM != NULL); - MsgLock(pM); - if(pM->pCSAPPNAME == NULL) - tryEmulateAPPNAME(pM); - MsgUnlock(pM); - return (pM->pCSAPPNAME == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSAPPNAME); -} - - /* rgerhards 2004-11-24: set PROCID in msg object */ rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID) @@ -1078,42 +1528,54 @@ rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID) ISOBJ_TYPE_assert(pMsg, msg); if(pMsg->pCSPROCID == NULL) { /* we need to obtain the object first */ - CHKiRet(rsCStrConstruct(&pMsg->pCSPROCID)); - rsCStrSetAllocIncrement(pMsg->pCSPROCID, 128); + CHKiRet(cstrConstruct(&pMsg->pCSPROCID)); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSPROCID, (uchar*) pszPROCID); + CHKiRet(cstrFinalize(pMsg->pCSPROCID)); finalize_it: RETiRet; } + +/* check if we have a procid, and, if not, try to aquire/emulate it. + * This must be called WITHOUT the message lock being held. + * rgerhards, 2009-06-26 + */ +static inline void preparePROCID(msg_t *pM, sbool bLockMutex) +{ + if(pM->pCSPROCID == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + /* re-query, things may have changed in the mean time... */ + if(pM->pCSPROCID == NULL) + aquirePROCIDFromTAG(pM); + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } +} + + +#if 0 /* rgerhards, 2005-11-24 */ -int getPROCIDLen(msg_t *pM) +static inline int getPROCIDLen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); - MsgLock(pM); - if(pM->pCSPROCID == NULL) - aquirePROCIDFromTAG(pM); - MsgUnlock(pM); + preparePROCID(pM, bLockMutex); return (pM->pCSPROCID == NULL) ? 1 : rsCStrLen(pM->pCSPROCID); } +#endif /* rgerhards, 2005-11-24 */ -char *getPROCID(msg_t *pM) +char *getPROCID(msg_t *pM, sbool bLockMutex) { - char* pszRet; - ISOBJ_TYPE_assert(pM, msg); - MsgLock(pM); - if(pM->pCSPROCID == NULL) - aquirePROCIDFromTAG(pM); - pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSPROCID); - MsgUnlock(pM); - return pszRet; + preparePROCID(pM, bLockMutex); + return (pM->pCSPROCID == NULL) ? "-" : (char*) cstrGetSzStrNoNULL(pM->pCSPROCID); } @@ -1126,7 +1588,6 @@ rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID) if(pMsg->pCSMSGID == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSMSGID)); - rsCStrSetAllocIncrement(pMsg->pCSMSGID, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSMSGID, (uchar*) pszMSGID); @@ -1135,49 +1596,50 @@ finalize_it: RETiRet; } -/* rgerhards, 2005-11-24 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getMSGIDLen(msg_t *pM) -{ - return (pM->pCSMSGID == NULL) ? 1 : rsCStrLen(pM->pCSMSGID); -} -#endif - /* rgerhards, 2005-11-24 */ -char *getMSGID(msg_t *pM) +static inline char *getMSGID(msg_t *pM) { return (pM->pCSMSGID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSMSGID); } -/* Set the TAG to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetTAG(). - * rgerhards 2004-11-19 +/* rgerhards 2009-06-12: set associated ruleset */ -void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) +void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) { assert(pMsg != NULL); - pMsg->iLenTAG = (pBuf == NULL) ? 0 : strlen((char*)pBuf); - pMsg->pszTAG = (uchar*) pBuf; + pMsg->pRuleset = pRuleset; } -/* rgerhards 2004-11-16: set TAG in msg object +/* set TAG in msg object + * (rewritten 2009-06-18 rgerhards) */ -void MsgSetTAG(msg_t *pMsg, char* pszTAG) +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) { + uchar *pBuf; assert(pMsg != NULL); - if(pMsg->pszTAG != NULL) - free(pMsg->pszTAG); - pMsg->iLenTAG = strlen(pszTAG); - if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) - memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); - else - dbgprintf("Could not allocate memory in MsgSetTAG()\n"); + + freeTAG(pMsg); + + pMsg->iLenTAG = lenBuf; + if(pMsg->iLenTAG < CONF_TAG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pBuf = pMsg->TAG.szBuf; + } else { + if((pBuf = (uchar*) MALLOC(pMsg->iLenTAG + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pBuf = pMsg->TAG.szBuf; + pMsg->iLenTAG = CONF_TAG_BUFSIZE - 1; + } else { + pMsg->TAG.pszTAG = pBuf; + } + } + + memcpy(pBuf, pszBuf, pMsg->iLenTAG); + pBuf[pMsg->iLenTAG] = '\0'; /* this also works with truncation! */ } @@ -1188,63 +1650,51 @@ void MsgSetTAG(msg_t *pMsg, char* pszTAG) * if there is a TAG and, if not, if it can emulate it. * rgerhards, 2005-11-24 */ -static void tryEmulateTAG(msg_t *pM) +static inline void tryEmulateTAG(msg_t *pM, sbool bLockMutex) { - int iTAGLen; - uchar *pBuf; + size_t lenTAG; + uchar bufTAG[CONF_TAG_MAXSIZE]; assert(pM != NULL); - if(pM->pszTAG != NULL) + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + if(pM->iLenTAG > 0) return; /* done, no need to emulate */ if(getProtocolVersion(pM) == 1) { - if(!strcmp(getPROCID(pM), "-")) { + if(!strcmp(getPROCID(pM, MUTEX_ALREADY_LOCKED), "-")) { /* no process ID, use APP-NAME only */ - MsgSetTAG(pM, getAPPNAME(pM)); + MsgSetTAG(pM, (uchar*) getAPPNAME(pM, MUTEX_ALREADY_LOCKED), getAPPNAMELen(pM, MUTEX_ALREADY_LOCKED)); } else { /* now we can try to emulate */ - iTAGLen = getAPPNAMELen(pM) + getPROCIDLen(pM) + 3; - if((pBuf = malloc(iTAGLen * sizeof(char))) == NULL) - return; /* nothing we can do */ - snprintf((char*)pBuf, iTAGLen, "%s[%s]", getAPPNAME(pM), getPROCID(pM)); - MsgAssignTAG(pM, pBuf); + lenTAG = snprintf((char*)bufTAG, CONF_TAG_MAXSIZE, "%s[%s]", + getAPPNAME(pM, MUTEX_ALREADY_LOCKED), getPROCID(pM, MUTEX_ALREADY_LOCKED)); + bufTAG[32] = '\0'; /* just to make sure... */ + MsgSetTAG(pM, bufTAG, lenTAG); } } + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); } -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getTAGLen(msg_t *pM) -{ - if(pM == NULL) - return 0; - else { - tryEmulateTAG(pM); - if(pM->pszTAG == NULL) - return 0; - else - return pM->iLenTAG; - } -} -#endif - - -char *getTAG(msg_t *pM) +static inline void +getTAG(msg_t *pM, uchar **ppBuf, int *piLen) { - char *ret; - - if(pM == NULL) - ret = ""; - else { - MsgLock(pM); - tryEmulateTAG(pM); - if(pM->pszTAG == NULL) - ret = ""; - else - ret = (char*) pM->pszTAG; - MsgUnlock(pM); + if(pM == NULL) { + *ppBuf = UCHAR_CONSTANT(""); + *piLen = 0; + } else { + if(pM->iLenTAG == 0) + tryEmulateTAG(pM, LOCK_MUTEX); + if(pM->iLenTAG == 0) { + *ppBuf = UCHAR_CONSTANT(""); + *piLen = 0; + } else { + *ppBuf = (pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG; + *piLen = pM->iLenTAG; + } } - return(ret); } @@ -1253,9 +1703,13 @@ int getHOSTNAMELen(msg_t *pM) if(pM == NULL) return 0; else - if(pM->pszHOSTNAME == NULL) - return 0; - else + if(pM->pszHOSTNAME == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) + return 0; + else + return prop.GetStringLen(pM->rcvFrom.pRcvFrom); + } else return pM->iLenHOSTNAME; } @@ -1265,48 +1719,42 @@ char *getHOSTNAME(msg_t *pM) if(pM == NULL) return ""; else - if(pM->pszHOSTNAME == NULL) - return ""; - else + if(pM->pszHOSTNAME == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) { + return ""; + } else { + uchar *psz; + int len; + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); + return (char*) psz; + } + } else { return (char*) pM->pszHOSTNAME; -} - - -uchar *getInputName(msg_t *pM) -{ - if(pM == NULL) - return (uchar*) ""; - else - if(pM->pszInputName == NULL) - return (uchar*) ""; - else - return pM->pszInputName; + } } uchar *getRcvFrom(msg_t *pM) { - if(pM == NULL) - return UCHAR_CONSTANT(""); - else - if(pM->pszRcvFrom == NULL) - return UCHAR_CONSTANT(""); - else - return pM->pszRcvFrom; -} - + uchar *psz; + int len; + BEGINfunc -uchar *getRcvFromIP(msg_t *pM) -{ - if(pM == NULL) - return (uchar*) ""; - else - if(pM->pszRcvFromIP == NULL) - return (uchar*) ""; + if(pM == NULL) { + psz = UCHAR_CONSTANT(""); + } else { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) + psz = UCHAR_CONSTANT(""); else - return pM->pszRcvFromIP; + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); + } + ENDfunc + return psz; } + /* rgerhards 2004-11-24: set STRUCTURED DATA in msg object */ rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData) @@ -1316,7 +1764,6 @@ rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData) if(pMsg->pCSStrucData == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSStrucData)); - rsCStrSetAllocIncrement(pMsg->pCSStrucData, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSStrucData, (uchar*) pszStrucData); @@ -1339,107 +1786,56 @@ static int getStructuredDataLen(msg_t *pM) /* get the "STRUCTURED-DATA" as sz string * rgerhards, 2005-11-24 */ -char *getStructuredData(msg_t *pM) +static inline char *getStructuredData(msg_t *pM) { return (pM->pCSStrucData == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSStrucData); } - -/* get the length of the "programname" sz string - * rgerhards, 2005-10-19 +/* check if we have a ProgramName, and, if not, try to aquire/emulate it. + * rgerhards, 2009-06-26 */ -int getProgramNameLen(msg_t *pM) +static inline void prepareProgramName(msg_t *pM, sbool bLockMutex) { - int iRet; + if(pM->pCSProgName == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); - assert(pM != NULL); - MsgLock(pM); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramNameLen()\n", iRet); - MsgUnlock(pM); - return 0; /* best we can do (consistent wiht what getProgramName() returns) */ - } - MsgUnlock(pM); + /* re-query as things might have changed during locking */ + if(pM->pCSProgName == NULL) + aquireProgramName(pM); - return (pM->pCSProgName == NULL) ? 0 : rsCStrLen(pM->pCSProgName); + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } } -/* get the "programname" as sz string +/* get the length of the "programname" sz string * rgerhards, 2005-10-19 */ -char *getProgramName(msg_t *pM) /* this is the non-locking version for internal use */ +int getProgramNameLen(msg_t *pM, sbool bLockMutex) { - int iRet; - char *pszRet; - assert(pM != NULL); - MsgLock(pM); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramName()\n", iRet); - pszRet = ""; /* best we can do */ - } else { - pszRet = (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); - } - - MsgUnlock(pM); - return pszRet; + prepareProgramName(pM, bLockMutex); + return (pM->pCSProgName == NULL) ? 0 : rsCStrLen(pM->pCSProgName); } -/* The code below was an approach without PTHREAD_MUTEX_RECURSIVE - * However, it turned out to be quite complex. So far, we use recursive - * locking, which is OK from a performance point of view, especially as - * we do not anticipate that multithreading msg objects is used often. - * However, we may re-think about using non-recursive locking and I leave this - * code in here to conserve the idea. -- rgerhards, 2008-01-05 - */ -#if 0 -static char *getProgramNameNoLock(msg_t *pM) /* this is the non-locking version for internal use */ -{ - int iRet; - assert(pM != NULL); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramName()\n", iRet); - return ""; /* best we can do */ - } - return (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); -} -char *getProgramName(msg_t *pM) /* this is the external callable version */ +/* get the "programname" as sz string + * rgerhards, 2005-10-19 + */ +uchar *getProgramName(msg_t *pM, sbool bLockMutex) { - char *pszRet; - - MsgLock(pM); - pszRet = getProgramNameNoLock(pM); - MsgUnlock(pM); - return pszRet; + prepareProgramName(pM, bLockMutex); + return (pM->pCSProgName == NULL) ? UCHAR_CONSTANT("") : rsCStrGetSzStrNoNULL(pM->pCSProgName); } -/* an alternative approach has been: */ -/* The macro below is used to generate external function definitions - * for such functions that may also be called internally (and thus have - * both a locking and non-locking implementation. Over time, we could - * reconsider how we handle that. -- rgerhards, 2008-01-05 - */ -#define EXT_LOCKED_FUNC(fName, ret) \ -ret fName(msg_t *pM) \ -{ \ - ret valRet; \ - MsgLock(pM); \ - valRet = fName##NoLock(pM); \ - MsgUnlock(pM); \ - return(valRet); \ -} -EXT_LOCKED_FUNC(getProgramName, char*) -/* in this approach, the external function is provided by the macro and - * needs not to be writen. - */ -#endif /* #if 0 -- saved code */ /* This function tries to emulate APPNAME if it is not present. Its * main use is when we have received a log record via legacy syslog and * now would like to send out the same one via syslog-protocol. + * MUST be called with the Msg Lock locked! */ static void tryEmulateAPPNAME(msg_t *pM) { @@ -1449,80 +1845,164 @@ static void tryEmulateAPPNAME(msg_t *pM) if(getProtocolVersion(pM) == 0) { /* only then it makes sense to emulate */ - MsgSetAPPNAME(pM, getProgramName(pM)); + MsgSetAPPNAME(pM, (char*)getProgramName(pM, MUTEX_ALREADY_LOCKED)); } } + +/* check if we have a APPNAME, and, if not, try to aquire/emulate it. + * This must be called WITHOUT the message lock being held. + * rgerhards, 2009-06-26 + */ +static inline void prepareAPPNAME(msg_t *pM, sbool bLockMutex) +{ + if(pM->pCSAPPNAME == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + + /* re-query as things might have changed during locking */ + if(pM->pCSAPPNAME == NULL) + tryEmulateAPPNAME(pM); + + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } +} + /* rgerhards, 2005-11-24 */ -static int getAPPNAMELen(msg_t *pM) +char *getAPPNAME(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); - if(pM->pCSAPPNAME == NULL) - tryEmulateAPPNAME(pM); - return (pM->pCSAPPNAME == NULL) ? 0 : rsCStrLen(pM->pCSAPPNAME); + prepareAPPNAME(pM, bLockMutex); + return (pM->pCSAPPNAME == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSAPPNAME); } -/* rgerhards 2008-09-10: set pszInputName in msg object +/* rgerhards, 2005-11-24 */ -void MsgSetInputName(msg_t *pMsg, uchar* pszInputName) +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex) { - assert(pMsg != NULL); - if(pMsg->pszInputName != NULL) - free(pMsg->pszInputName); - - pMsg->iLenInputName = ustrlen(pszInputName); - if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { - memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); - } + assert(pM != NULL); + prepareAPPNAME(pM, bLockMutex); + return (pM->pCSAPPNAME == NULL) ? 0 : rsCStrLen(pM->pCSAPPNAME); } -/* rgerhards 2004-11-16: set pszRcvFrom in msg object +/* rgerhards 2008-09-10: set pszInputName in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-16 */ -void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom) +void MsgSetInputName(msg_t *pThis, prop_t *inputName) { - assert(pMsg != NULL); - free(pMsg->pszRcvFrom); + assert(pThis != NULL); - pMsg->iLenRcvFrom = ustrlen(pszRcvFrom); - if((pMsg->pszRcvFrom = malloc(pMsg->iLenRcvFrom + 1)) != NULL) { - memcpy(pMsg->pszRcvFrom, pszRcvFrom, pMsg->iLenRcvFrom + 1); - } + prop.AddRef(inputName); + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); + pThis->pInputName = inputName; } -/* rgerhards 2005-05-16: set pszRcvFromIP in msg object */ +/* Set the pfrominet socket store, so that we can obtain the peer at some + * later time. Note that we do not check if pRcvFrom is already set, so this + * function must only be called during message creation. + * NOTE: msgFlags is NOT set. While this is somewhat a violation of layers, + * it is done because it gains us some performance. So the caller must make + * sure the message flags are properly maintained. For all current callers, + * this is always the case and without extra effort required. + * rgerhards, 2009-11-17 + */ rsRetVal -MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP) -{ +msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa){ DEFiRet; - assert(pMsg != NULL); - if(pMsg->pszRcvFromIP != NULL) { - free(pMsg->pszRcvFromIP); - pMsg->iLenRcvFromIP = 0; - } + assert(pThis->rcvFrom.pRcvFrom == NULL); + + CHKmalloc(pThis->rcvFrom.pfrominet = malloc(sizeof(struct sockaddr_storage))); + memcpy(pThis->rcvFrom.pfrominet, sa, sizeof(struct sockaddr_storage)); - CHKmalloc(pMsg->pszRcvFromIP = (uchar*)strdup((char*)pszRcvFromIP)); - pMsg->iLenRcvFromIP = strlen((char*)pszRcvFromIP); finalize_it: RETiRet; } -/* Set the HOSTNAME to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetHOSTNAME(). - * rgerhards 2004-11-19 +/* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-30 */ -void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) +void MsgSetRcvFrom(msg_t *pThis, prop_t *new) { - assert(pMsg != NULL); - assert(pBuf != NULL); - if(pMsg->pszHOSTNAME != NULL) - free(pMsg->pszHOSTNAME); - pMsg->iLenHOSTNAME = strlen(pBuf); - pMsg->pszHOSTNAME = (uchar*) pBuf; + assert(pThis != NULL); + + prop.AddRef(new); + if(pThis->msgFlags & NEEDS_DNSRESOL) { + if(pThis->rcvFrom.pfrominet != NULL) + free(pThis->rcvFrom.pfrominet); + pThis->msgFlags &= ~NEEDS_DNSRESOL; + } else { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } + pThis->rcvFrom.pRcvFrom = new; +} + + +/* This is used to set the property via a string. This function should not be + * called if there is a reliable way for a caller to make sure that the + * same name can be used across multiple messages. However, if it can not + * ensure that, calling this function is the second best thing, because it + * will re-use the previously created property if it contained the same + * name (but it works only for the immediate previous). + * rgerhards, 2009-06-31 + */ +void MsgSetRcvFromStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp) +{ + assert(pThis != NULL); + assert(ppProp != NULL); + + prop.CreateOrReuseStringProp(ppProp, psz, len); + MsgSetRcvFrom(pThis, *ppProp); +} + + +/* set RcvFromIP name in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-30 + */ +rsRetVal MsgSetRcvFromIP(msg_t *pThis, prop_t *new) +{ + assert(pThis != NULL); + + BEGINfunc + prop.AddRef(new); + if(pThis->pRcvFromIP != NULL) + prop.Destruct(&pThis->pRcvFromIP); + pThis->pRcvFromIP = new; + ENDfunc + return RS_RET_OK; +} + + +/* This is used to set the property via a string. This function should not be + * called if there is a reliable way for a caller to make sure that the + * same name can be used across multiple messages. However, if it can not + * ensure that, calling this function is the second best thing, because it + * will re-use the previously created property if it contained the same + * name (but it works only for the immediate previous). + * rgerhards, 2009-06-31 + */ +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp) +{ + DEFiRet; + assert(pThis != NULL); + + CHKiRet(prop.CreateOrReuseStringProp(ppProp, psz, len)); + MsgSetRcvFromIP(pThis, *ppProp); + +finalize_it: + RETiRet; } @@ -1536,83 +2016,118 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) * we need it. The rest of the code already knows how to handle an * unset HOSTNAME. */ -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) +void MsgSetHOSTNAME(msg_t *pThis, uchar* pszHOSTNAME, int lenHOSTNAME) { - assert(pMsg != NULL); - free(pMsg->pszHOSTNAME); + assert(pThis != NULL); - pMsg->iLenHOSTNAME = ustrlen(pszHOSTNAME); - if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) - memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); - else - DBGPRINTF("Could not allocate memory in MsgSetHOSTNAME()\n"); + freeHOSTNAME(pThis); + + pThis->iLenHOSTNAME = lenHOSTNAME; + if(pThis->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pThis->pszHOSTNAME = pThis->szHOSTNAME; + } else if((pThis->pszHOSTNAME = (uchar*) MALLOC(pThis->iLenHOSTNAME + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pThis->pszHOSTNAME = pThis->szHOSTNAME; + pThis->iLenHOSTNAME = CONF_HOSTNAME_BUFSIZE - 1; + } + + memcpy(pThis->pszHOSTNAME, pszHOSTNAME, pThis->iLenHOSTNAME); + pThis->pszHOSTNAME[pThis->iLenHOSTNAME] = '\0'; /* this also works with truncation! */ } -/* Set the UxTradMsg to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetUxTradMsg(). - * rgerhards 2004-11-19 +/* set the offset of the MSG part into the raw msg buffer + * Note that the offset may be higher than the length of the raw message + * (exactly by one). This can happen if we have a message that does not + * contain any MSG part. */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static void MsgAssignUxTradMsg(msg_t *pMsg, char *pBuf) +void MsgSetMSGoffs(msg_t *pMsg, short offs) { - assert(pMsg != NULL); - assert(pBuf != NULL); - pMsg->iLenUxTradMsg = strlen(pBuf); - pMsg->pszUxTradMsg = pBuf; + ISOBJ_TYPE_assert(pMsg, msg); + pMsg->offMSG = offs; + if(offs > pMsg->iLenRawMsg) { + assert(offs - 1 == pMsg->iLenRawMsg); + pMsg->iLenMSG = 0; + } else { + pMsg->iLenMSG = pMsg->iLenRawMsg - offs; + } } -#endif -/* rgerhards 2004-11-17: set the traditional Unix message in msg object +/* replace the MSG part of a message. The update actually takes place inside + * rawmsg. + * There are two cases: either the new message will be larger than the new msg + * or it will be less than or equal. If it is less than or equal, we can utilize + * the previous message buffer. If it is larger, we can utilize the msg_t-included + * message buffer if it fits in there. If this is not the case, we need to alloc + * a new, larger, chunk and copy over the data to it. Note that this function is + * (hopefully) relatively seldom being called, so some performance impact is + * uncritical. In any case, pszMSG is copied, so if it was dynamically allocated, + * the caller is responsible for freeing it. + * rgerhards, 2009-06-23 */ -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg) +rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG) { - assert(pMsg != NULL); - assert(pszUxTradMsg != NULL); - pMsg->iLenUxTradMsg = strlen(pszUxTradMsg); - if(pMsg->pszUxTradMsg != NULL) - free(pMsg->pszUxTradMsg); - if((pMsg->pszUxTradMsg = malloc(pMsg->iLenUxTradMsg + 1)) != NULL) - memcpy(pMsg->pszUxTradMsg, pszUxTradMsg, pMsg->iLenUxTradMsg + 1); - else - dbgprintf("Could not allocate memory for pszUxTradMsg buffer."); + int lenNew; + uchar *bufNew; + DEFiRet; + ISOBJ_TYPE_assert(pThis, msg); + assert(pszMSG != NULL); + + lenNew = pThis->iLenRawMsg + lenMSG - pThis->iLenMSG; + if(lenMSG > pThis->iLenMSG && lenNew >= CONF_RAWMSG_BUFSIZE) { + /* we have lost our "bet" and need to alloc a new buffer ;) */ + CHKmalloc(bufNew = MALLOC(lenNew + 1)); + memcpy(bufNew, pThis->pszRawMsg, pThis->offMSG); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); + pThis->pszRawMsg = bufNew; + } - return(0); + if(lenMSG > 0) + memcpy(pThis->pszRawMsg + pThis->offMSG, pszMSG, lenMSG); + pThis->pszRawMsg[lenNew] = '\0'; /* this also works with truncation! */ + pThis->iLenRawMsg = lenNew; + pThis->iLenMSG = lenMSG; + +finalize_it: + RETiRet; } -/* rgerhards 2004-11-09: set MSG in msg object +/* set raw message in message object. Size of message is provided. + * rgerhards, 2009-06-16 */ -void MsgSetMSG(msg_t *pMsg, char* pszMSG) +void MsgSetRawMsg(msg_t *pThis, char* pszRawMsg, size_t lenMsg) { - assert(pMsg != NULL); - assert(pszMSG != NULL); + assert(pThis != NULL); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); - if(pMsg->pszMSG != NULL) - free(pMsg->pszMSG); + pThis->iLenRawMsg = lenMsg; + if(pThis->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pThis->pszRawMsg = pThis->szRawMsg; + } else if((pThis->pszRawMsg = (uchar*) MALLOC(pThis->iLenRawMsg + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pThis->pszRawMsg = pThis->szRawMsg; + pThis->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; + } - pMsg->iLenMSG = strlen(pszMSG); - if((pMsg->pszMSG = (uchar*) malloc(pMsg->iLenMSG + 1)) != NULL) - memcpy(pMsg->pszMSG, pszMSG, pMsg->iLenMSG + 1); - else - dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); + memcpy(pThis->pszRawMsg, pszRawMsg, pThis->iLenRawMsg); + pThis->pszRawMsg[pThis->iLenRawMsg] = '\0'; /* this also works with truncation! */ } -/* rgerhards 2004-11-11: set RawMsg in msg object + +/* set raw message in message object. Size of message is not provided. This + * function should only be used when it is unavoidable (and over time we should + * try to remove it altogether). + * rgerhards, 2009-06-16 */ -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg) { - assert(pMsg != NULL); - if(pMsg->pszRawMsg != NULL) - free(pMsg->pszRawMsg); - - pMsg->iLenRawMsg = strlen(pszRawMsg); - if((pMsg->pszRawMsg = (uchar*) malloc(pMsg->iLenRawMsg + 1)) != NULL) - memcpy(pMsg->pszRawMsg, pszRawMsg, pMsg->iLenRawMsg + 1); - else - dbgprintf("Could not allocate memory for pszRawMsg buffer."); + MsgSetRawMsg(pMsg, pszRawMsg, strlen(pszRawMsg)); } @@ -1626,15 +2141,11 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) */ char *textpri(char *pRes, size_t pResLen, int pri) { - syslogCODE *c_pri, *c_fac; - assert(pRes != NULL); assert(pResLen > 0); - for (c_fac = rs_facilitynames; c_fac->c_name && !(c_fac->c_val == LOG_FAC(pri)<<3); c_fac++); - for (c_pri = rs_prioritynames; c_pri->c_name && !(c_pri->c_val == LOG_PRI(pri)); c_pri++); - - snprintf (pRes, pResLen, "%s.%s<%d>", c_fac->c_name, c_pri->c_name, pri); + snprintf(pRes, pResLen, "%s.%s<%d>", syslog_fac_names[LOG_FAC(pri)], + syslog_severity_names[LOG_PRI(pri)], pri); return pRes; } @@ -1654,7 +2165,7 @@ static uchar *getNOW(eNOWType eNow) uchar *pBuf; struct syslogTime t; - if((pBuf = (uchar*) malloc(sizeof(uchar) * tmpBUFSIZE)) == NULL) { + if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { return NULL; } @@ -1728,140 +2239,177 @@ static uchar *getNOW(eNOWType eNow) * be used in selector line processing. * rgerhards 2005-09-15 */ -char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed) -{ - uchar *pName; - char *pRes; /* result pointer */ - char *pBufStart; - char *pBuf; +/* a quick helper to save some writing: */ +#define RET_OUT_OF_MEMORY { *pbMustBeFreed = 0;\ + *pPropLen = sizeof("**OUT OF MEMORY**") - 1; \ + return(UCHAR_CONSTANT("**OUT OF MEMORY**"));} +uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, + propid_t propID, size_t *pPropLen, + unsigned short *pbMustBeFreed) +{ + uchar *pRes; /* result pointer */ + int bufLen = -1; /* length of string or -1, if not known */ + uchar *pBufStart; + uchar *pBuf; int iLen; short iOffs; + BEGINfunc + assert(pMsg != NULL); + assert(pbMustBeFreed != NULL); + #ifdef FEATURE_REGEXP /* Variables necessary for regular expression matching */ size_t nmatch = 10; regmatch_t pmatch[10]; #endif - assert(pMsg != NULL); - assert(pbMustBeFreed != NULL); - - if(pCSPropName == NULL) { - assert(pTpe != NULL); - pName = pTpe->data.field.pPropRepl; - } else { - pName = rsCStrGetSzStrNoNULL(pCSPropName); - } *pbMustBeFreed = 0; - /* sometimes there are aliases to the original MonitoWare - * property names. These come after || in the ifs below. */ - if(!strcmp((char*) pName, "msg")) { - pRes = getMSG(pMsg); - } else if(!strcmp((char*) pName, "rawmsg")) { - pRes = getRawMsg(pMsg); - } else if(!strcmp((char*) pName, "uxtradmsg")) { - pRes = getUxTradMsg(pMsg); - } else if(!strcmp((char*) pName, "inputname")) { - pRes = (char*) getInputName(pMsg); - } else if(!strcmp((char*) pName, "fromhost")) { - pRes = (char*) getRcvFrom(pMsg); - } else if(!strcmp((char*) pName, "fromhost-ip")) { - pRes = (char*) getRcvFromIP(pMsg); - } else if(!strcmp((char*) pName, "source") || !strcmp((char*) pName, "hostname")) { - pRes = getHOSTNAME(pMsg); - } else if(!strcmp((char*) pName, "syslogtag")) { - pRes = getTAG(pMsg); - } else if(!strcmp((char*) pName, "pri")) { - pRes = getPRI(pMsg); - } else if(!strcmp((char*) pName, "pri-text")) { - pBuf = malloc(20 * sizeof(char)); - if(pBuf == NULL) { + switch(propID) { + case PROP_MSG: + pRes = getMSG(pMsg); + bufLen = getMSGLen(pMsg); + break; + case PROP_TIMESTAMP: + pRes = (uchar*)getTimeReported(pMsg, pTpe->data.field.eDateFormat); + break; + case PROP_HOSTNAME: + pRes = (uchar*)getHOSTNAME(pMsg); + bufLen = getHOSTNAMELen(pMsg); + break; + case PROP_SYSLOGTAG: + getTAG(pMsg, &pRes, &bufLen); + break; + case PROP_RAWMSG: + getRawMsg(pMsg, &pRes, &bufLen); + break; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + case PROP_UXTRADMSG: + pRes = getUxTradMsg(pMsg); + break; + */ + case PROP_INPUTNAME: + getInputName(pMsg, &pRes, &bufLen); + break; + case PROP_FROMHOST: + pRes = getRcvFrom(pMsg); + break; + case PROP_FROMHOST_IP: + pRes = getRcvFromIP(pMsg); + break; + case PROP_PRI: + pRes = (uchar*)getPRI(pMsg); + break; + case PROP_PRI_TEXT: + pBuf = MALLOC(20 * sizeof(uchar)); + if(pBuf == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + pRes = (uchar*)textpri((char*)pBuf, 20, getPRIi(pMsg)); + } + break; + case PROP_IUT: + pRes = UCHAR_CONSTANT("1"); /* always 1 for syslog messages (a MonitorWare thing;)) */ + bufLen = 1; + break; + case PROP_SYSLOGFACILITY: + pRes = (uchar*)getFacility(pMsg); + break; + case PROP_SYSLOGFACILITY_TEXT: + pRes = (uchar*)getFacilityStr(pMsg); + break; + case PROP_SYSLOGSEVERITY: + pRes = (uchar*)getSeverity(pMsg); + break; + case PROP_SYSLOGSEVERITY_TEXT: + pRes = (uchar*)getSeverityStr(pMsg); + break; + case PROP_TIMEGENERATED: + pRes = (uchar*)getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); + break; + case PROP_PROGRAMNAME: + pRes = getProgramName(pMsg, LOCK_MUTEX); + break; + case PROP_PROTOCOL_VERSION: + pRes = (uchar*)getProtocolVersionString(pMsg); + break; + case PROP_STRUCTURED_DATA: + pRes = (uchar*)getStructuredData(pMsg); + break; + case PROP_APP_NAME: + pRes = (uchar*)getAPPNAME(pMsg, LOCK_MUTEX); + break; + case PROP_PROCID: + pRes = (uchar*)getPROCID(pMsg, LOCK_MUTEX); + break; + case PROP_MSGID: + pRes = (uchar*)getMSGID(pMsg); + break; + case PROP_SYS_NOW: + if((pRes = getNOW(NOW_NOW)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_YEAR: + if((pRes = getNOW(NOW_YEAR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MONTH: + if((pRes = getNOW(NOW_MONTH)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_DAY: + if((pRes = getNOW(NOW_DAY)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_HOUR: + if((pRes = getNOW(NOW_HOUR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_HHOUR: + if((pRes = getNOW(NOW_HHOUR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_QHOUR: + if((pRes = getNOW(NOW_QHOUR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MINUTE: + if((pRes = getNOW(NOW_MINUTE)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MYHOSTNAME: + pRes = glbl.GetLocalHostName(); + break; + default: + /* there is no point in continuing, we may even otherwise render the + * error message unreadable. rgerhards, 2007-07-10 + */ + dbgprintf("invalid property id: '%d'\n", propID); *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } else { - *pbMustBeFreed = 1; - pRes = textpri(pBuf, 20, getPRIi(pMsg)); - } - } else if(!strcmp((char*) pName, "iut")) { - pRes = "1"; /* always 1 for syslog messages (a MonitorWare thing;)) */ - } else if(!strcmp((char*) pName, "syslogfacility")) { - pRes = getFacility(pMsg); - } else if(!strcmp((char*) pName, "syslogfacility-text")) { - pRes = getFacilityStr(pMsg); - } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { - pRes = getSeverity(pMsg); - } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { - pRes = getSeverityStr(pMsg); - } else if(!strcmp((char*) pName, "timegenerated")) { - pRes = getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "timereported") - || !strcmp((char*) pName, "timestamp")) { - pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "programname")) { - pRes = getProgramName(pMsg); - } else if(!strcmp((char*) pName, "protocol-version")) { - pRes = getProtocolVersionString(pMsg); - } else if(!strcmp((char*) pName, "structured-data")) { - pRes = getStructuredData(pMsg); - } else if(!strcmp((char*) pName, "app-name")) { - pRes = getAPPNAME(pMsg); - } else if(!strcmp((char*) pName, "procid")) { - pRes = getPROCID(pMsg); - } else if(!strcmp((char*) pName, "msgid")) { - pRes = getMSGID(pMsg); - /* here start system properties (those, that do not relate to the message itself */ - } else if(!strcmp((char*) pName, "$now")) { - if((pRes = (char*) getNOW(NOW_NOW)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$year")) { - if((pRes = (char*) getNOW(NOW_YEAR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$month")) { - if((pRes = (char*) getNOW(NOW_MONTH)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$day")) { - if((pRes = (char*) getNOW(NOW_DAY)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$hour")) { - if((pRes = (char*) getNOW(NOW_HOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$hhour")) { - if((pRes = (char*) getNOW(NOW_HHOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$qhour")) { - if((pRes = (char*) getNOW(NOW_QHOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$minute")) { - if((pRes = (char*) getNOW(NOW_MINUTE)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$myhostname")) { - pRes = (char*) glbl.GetLocalHostName(); - } else { - /* there is no point in continuing, we may even otherwise render the - * error message unreadable. rgerhards, 2007-07-10 - */ - dbgprintf("invalid property name: '%s'\n", pName); - return "**INVALID PROPERTY NAME**"; + *pPropLen = sizeof("**INVALID PROPERTY NAME**") - 1; + return UCHAR_CONSTANT("**INVALID PROPERTY NAME**"); } + /* If we did not receive a template pointer, we are already done... */ if(pTpe == NULL) { return pRes; @@ -1878,8 +2426,8 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.has_fields == 1) { size_t iCurrFld; - char *pFld; - char *pFldEnd; + uchar *pFld; + uchar *pFldEnd; /* first, skip to the field in question. The field separator * is always one character and is stored in the template entry. */ @@ -1912,15 +2460,15 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* we got our end pointer, now do the copy */ /* TODO: code copied from below, this is a candidate for a separate function */ iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } /* now copy */ memcpy(pBuf, pFld, iLen); + bufLen = iLen; pBuf[iLen] = '\0'; /* terminate it */ if(*pbMustBeFreed == 1) free(pRes); @@ -1933,12 +2481,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); *pbMustBeFreed = 0; - return "**FIELD NOT FOUND**"; + *pPropLen = sizeof("**FIELD NOT FOUND**") - 1; + return UCHAR_CONSTANT("**FIELD NOT FOUND**"); } } else if(pTpe->data.field.iFromPos != 0 || pTpe->data.field.iToPos != 0) { /* we need to obtain a private copy */ int iFrom, iTo; - char *pSb; + uchar *pSb; iFrom = pTpe->data.field.iFromPos; iTo = pTpe->data.field.iToPos; /* need to zero-base to and from (they are 1-based!) */ @@ -1946,42 +2495,60 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, --iFrom; if(iTo > 0) --iTo; - iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); - if(pBuf == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } - pSb = pRes; - if(iFrom) { - /* skip to the start of the substring (can't do pointer arithmetic - * because the whole string might be smaller!!) - */ - while(*pSb && iFrom) { - --iFrom; + if(bufLen == -1) + bufLen = ustrlen(pRes); + if(iFrom == 0 && iTo >= bufLen) { + /* in this case, the requested string is a superset of what we already have, + * so there is no need to do any processing. This is a frequent case for size-limited + * fields like TAG in the default forwarding template (so it is a useful optimization + * to check for this condition ;)). -- rgerhards, 2009-07-09 + */ + ; /*DO NOTHING*/ + } else { + iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); + if(pBuf == NULL) { + if(*pbMustBeFreed == 1) + free(pRes); + RET_OUT_OF_MEMORY; + } + pSb = pRes; + if(iFrom) { + /* skip to the start of the substring (can't do pointer arithmetic + * because the whole string might be smaller!!) + */ + while(*pSb && iFrom) { + --iFrom; + ++pSb; + } + } + /* OK, we are at the begin - now let's copy... */ + bufLen = iLen; + while(*pSb && iLen) { + *pBuf++ = *pSb; ++pSb; + --iLen; } + *pBuf = '\0'; + bufLen -= iLen; /* subtract remaining length if the string was smaller! */ + if(*pbMustBeFreed == 1) + free(pRes); + pRes = pBufStart; + *pbMustBeFreed = 1; } - /* OK, we are at the begin - now let's copy... */ - while(*pSb && iLen) { - *pBuf++ = *pSb; - ++pSb; - --iLen; - } - *pBuf = '\0'; - if(*pbMustBeFreed == 1) - free(pRes); - pRes = pBufStart; - *pbMustBeFreed = 1; #ifdef FEATURE_REGEXP } else { /* Check for regular expressions */ if (pTpe->data.field.has_regex != 0) { - if (pTpe->data.field.has_regex == 2) + if (pTpe->data.field.has_regex == 2) { /* Could not compile regex before! */ - return "**NO MATCH** **BAD REGULAR EXPRESSION**"; + if (*pbMustBeFreed == 1) { + free(pRes); + *pbMustBeFreed = 0; + } + *pPropLen = sizeof("**NO MATCH** **BAD REGULAR EXPRESSION**") - 1; + return UCHAR_CONSTANT("**NO MATCH** **BAD REGULAR EXPRESSION**"); + } dbgprintf("string to match for regex is: %s\n", pRes); @@ -1994,7 +2561,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ while(!bFound) { int iREstat; - iREstat = regexp.regexec(&pTpe->data.field.re, pRes + iOffs, nmatch, pmatch, 0); + iREstat = regexp.regexec(&pTpe->data.field.re, (char*)(pRes + iOffs), nmatch, pmatch, 0); dbgprintf("regexec return is %d\n", iREstat); if(iREstat == 0) { if(pmatch[0].rm_so == -1) { @@ -2021,12 +2588,16 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) - return "**NO MATCH**"; - else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) - return "0"; - else - return ""; + if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) { + bufLen = sizeof("**NO MATCH**") - 1; + pRes = UCHAR_CONSTANT("**NO MATCH**"); + } else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) { + bufLen = 1; + pRes = UCHAR_CONSTANT("0"); + } else { + bufLen = 0; + pRes = UCHAR_CONSTANT(""); + } } } else { /* Match- but did it match the one we wanted? */ @@ -2037,29 +2608,35 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) - return "**NO MATCH**"; - else - return ""; + if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) { + bufLen = sizeof("**NO MATCH**") - 1; + pRes = UCHAR_CONSTANT("**NO MATCH**"); + } else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) { + bufLen = 1; + pRes = UCHAR_CONSTANT("0"); + } else { + bufLen = 0; + pRes = UCHAR_CONSTANT(""); + } } } /* OK, we have a usable match - we now need to malloc pB */ int iLenBuf; - char *pB; + uchar *pB; iLenBuf = pmatch[pTpe->data.field.iSubMatchToUse].rm_eo - pmatch[pTpe->data.field.iSubMatchToUse].rm_so; - pB = (char *) malloc((iLenBuf + 1) * sizeof(char)); + pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if (pB == NULL) { if (*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY ALLOCATING pBuf**"; + RET_OUT_OF_MEMORY; } /* Lets copy the matched substring to the buffer */ memcpy(pB, pRes + iOffs + pmatch[pTpe->data.field.iSubMatchToUse].rm_so, iLenBuf); + bufLen = iLenBuf; pB[iLenBuf] = '\0';/* terminate string, did not happen before */ if (*pbMustBeFreed == 1) @@ -2077,7 +2654,8 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - return "***REGEXP NOT AVAILABLE***"; + *pPropLen = sizeof("***REGEXP NOT AVAILABLE***") - 1; + return UCHAR_CONSTANT("***REGEXP NOT AVAILABLE***"); } } #endif /* #ifdef FEATURE_REGEXP */ @@ -2085,28 +2663,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* now check if we need to do our "SP if first char is non-space" hack logic */ if(*pRes && pTpe->data.field.options.bSPIffNo1stSP) { - char *pB; - uchar cFirst = *pRes; - /* here, we always destruct the buffer and return a new one */ - pB = (char *) malloc(2 * sizeof(char)); - if(pB == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } - pRes = pB; - *pbMustBeFreed = 1; - - if(cFirst == ' ') { - /* if we have a SP, we must return an empty string */ - *pRes = '\0'; /* empty */ - } else { - /* if it is no SP, we need to return one */ - *pRes = ' '; - *(pRes+1) = '\0'; - } + uchar cFirst = *pRes; /* save first char */ + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (cFirst == ' ') ? UCHAR_CONSTANT("") : UCHAR_CONSTANT(" "); + bufLen = (cFirst == ' ') ? 0 : 1; + *pbMustBeFreed = 0; } if(*pRes) { @@ -2115,21 +2678,21 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.eCaseConv != tplCaseConvNo) { /* we need to obtain a private copy */ - int iBufLen = strlen(pRes); - char *pBStart; - char *pB; - char *pSrc; - pBStart = pB = malloc((iBufLen + 1) * sizeof(char)); + if(bufLen == -1) + bufLen = ustrlen(pRes); + uchar *pBStart; + uchar *pB; + uchar *pSrc; + pBStart = pB = MALLOC((bufLen + 1) * sizeof(char)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } pSrc = pRes; while(*pSrc) { *pB++ = (pTpe->data.field.eCaseConv == tplCaseConvUpper) ? - (char)toupper((int)*pSrc) : (char)tolower((int)*pSrc); + (uchar)toupper((int)*pSrc) : (uchar)tolower((int)*pSrc); /* currently only these two exist */ ++pSrc; } @@ -2153,10 +2716,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.options.bDropCC) { int iLenBuf = 0; - char *pSrc = pRes; - char *pDstStart; - char *pDst; - char bDropped = 0; + uchar *pSrc = pRes; + uchar *pDstStart; + uchar *pDst; + uchar bDropped = 0; while(*pSrc) { if(!iscntrl((int) *pSrc++)) @@ -2166,12 +2729,11 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(!iscntrl((int) *pSrc)) @@ -2181,12 +2743,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = iLenBuf; *pbMustBeFreed = 1; } } else if(pTpe->data.field.options.bSpaceCC) { - char *pSrc; - char *pDstStart; - char *pDst; + uchar *pSrc; + uchar *pDstStart; + uchar *pDst; if(*pbMustBeFreed == 1) { /* in this case, we already work on dynamic @@ -2199,12 +2762,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst = ' '; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = ustrlen(pRes); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(iscntrl((int) *pSrc)) @@ -2224,7 +2788,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ int iNumCC = 0; int iLenBuf = 0; - char *pB; + uchar *pB; for(pB = pRes ; *pB ; ++pB) { ++iLenBuf; @@ -2234,21 +2798,20 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(iNumCC > 0) { /* if 0, there is nothing to escape, so we are done */ /* OK, let's do the escaping... */ - char *pBStart; - char szCCEsc[8]; /* buffer for escape sequence */ + uchar *pBStart; + uchar szCCEsc[8]; /* buffer for escape sequence */ int i; iLenBuf += iNumCC * 4; - pBStart = pB = malloc((iLenBuf + 1) * sizeof(char)); + pBStart = pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } while(*pRes) { if(iscntrl((int) *pRes)) { - snprintf(szCCEsc, sizeof(szCCEsc), "#%3.3d", *pRes); + snprintf((char*)szCCEsc, sizeof(szCCEsc), "#%3.3d", *pRes); for(i = 0 ; i < 4 ; ++i) *pB++ = szCCEsc[i]; } else { @@ -2260,6 +2823,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } } @@ -2271,10 +2835,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pTpe->data.field.options.bSecPathDrop || pTpe->data.field.options.bSecPathReplace) { if(pTpe->data.field.options.bSecPathDrop) { int iLenBuf = 0; - char *pSrc = pRes; - char *pDstStart; - char *pDst; - char bDropped = 0; + uchar *pSrc = pRes; + uchar *pDstStart; + uchar *pDst; + uchar bDropped = 0; while(*pSrc) { if(*pSrc++ != '/') @@ -2284,12 +2848,11 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(*pSrc != '/') @@ -2299,12 +2862,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = -1; /* TODO: can we do better? */ *pbMustBeFreed = 1; } } else { - char *pSrc; - char *pDstStart; - char *pDst; + uchar *pSrc; + uchar *pDstStart; + uchar *pDst; if(*pbMustBeFreed == 1) { /* here, again, we can modify the string as we already obtained @@ -2317,12 +2881,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst++ = '_'; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = ustrlen(pRes); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(*pSrc == '/') @@ -2342,44 +2907,48 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* check for "." and ".." (note the parenthesis in the if condition!) */ if((*pRes == '.') && (*(pRes + 1) == '\0' || (*(pRes + 1) == '.' && *(pRes + 2) == '\0'))) { - char *pTmp = pRes; + uchar *pTmp = pRes; if(*(pRes + 1) == '\0') - pRes = "_"; + pRes = UCHAR_CONSTANT("_"); else - pRes = "_.";; + pRes = UCHAR_CONSTANT("_.");; if(*pbMustBeFreed == 1) free(pTmp); *pbMustBeFreed = 0; } else if(*pRes == '\0') { if(*pbMustBeFreed == 1) free(pRes); - pRes = "_"; + pRes = UCHAR_CONSTANT("_"); + bufLen = 1; *pbMustBeFreed = 0; } } /* Now drop last LF if present (pls note that this must not be done - * if bEscapeCC was set! + * if bEscapeCC was set)! */ if(pTpe->data.field.options.bDropLastLF && !pTpe->data.field.options.bEscapeCC) { - int iLn = strlen(pRes); - char *pB; + int iLn; + uchar *pB; + if(bufLen == -1) + bufLen = ustrlen(pRes); + iLn = bufLen; if(iLn > 0 && *(pRes + iLn - 1) == '\n') { /* we have a LF! */ /* check if we need to obtain a private copy */ if(*pbMustBeFreed == 0) { /* ok, original copy, need a private one */ - pB = malloc((iLn + 1) * sizeof(char)); + pB = MALLOC((iLn + 1) * sizeof(uchar)); if(pB == NULL) { - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } memcpy(pB, pRes, iLn - 1); pRes = pB; *pbMustBeFreed = 1; } *(pRes + iLn - 1) = '\0'; /* drop LF ;) */ + --bufLen; } } @@ -2390,19 +2959,23 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.options.bCSV) { /* we need to obtain a private copy, as we need to at least add the double quotes */ - int iBufLen = strlen(pRes); - char *pBStart; - char *pDst; - char *pSrc; + int iBufLen; + int i; + uchar *pBStart; + uchar *pDst; + uchar *pSrc; + if(bufLen == -1) + bufLen = ustrlen(pRes); + iBufLen = bufLen; /* the malloc may be optimized, we currently use the worst case... */ - pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(char)); + pBStart = pDst = MALLOC((2 * iBufLen + 3) * sizeof(uchar)); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } pSrc = pRes; + i = 0; *pDst++ = '"'; /* starting quote */ while(*pSrc) { if(*pSrc == '"') @@ -2414,10 +2987,15 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } - /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ + if(bufLen == -1) + bufLen = ustrlen(pRes); + *pPropLen = bufLen; + + ENDfunc return(pRes); } @@ -2432,8 +3010,10 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) { DEFiRet; var_t *pVar; + size_t propLen; uchar *pszProp = NULL; cstr_t *pstrProp; + propid_t propid; unsigned short bMustBeFreed = 0; ISOBJ_TYPE_assert(pThis, msg); @@ -2445,7 +3025,9 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) CHKiRet(var.ConstructFinalize(pVar)); /* always call MsgGetProp() without a template specifier */ - pszProp = (uchar*) MsgGetProp(pThis, NULL, pstrPropName, &bMustBeFreed); + /* TODO: optimize propNameToID() call -- rgerhards, 2009-06-26 */ + propNameToID(pstrPropName, &propid); + pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, &propLen, &bMustBeFreed); /* now create a string object out of it and hand that over to the var */ CHKiRet(rsCStrConstructFromszStr(&pstrProp, pszProp)); @@ -2460,8 +3042,6 @@ finalize_it: RETiRet; } - - /* This function can be used as a generic way to set properties. * We have to handle a lot of legacy, so our return value is not always * 100% correct (called functions do not always provide one, should @@ -2471,6 +3051,9 @@ finalize_it: #define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) { + prop_t *myProp; + prop_t *propRcvFrom = NULL; + prop_t *propRcvFromIP = NULL; DEFiRet; ISOBJ_TYPE_assert(pThis, msg); @@ -2484,22 +3067,34 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) pThis->iFacility = pProp->val.num; } else if(isProp("msgFlags")) { pThis->msgFlags = pProp->val.num; + } else if(isProp("offMSG")) { + MsgSetMSGoffs(pThis, pProp->val.num); } else if(isProp("pszRawMsg")) { - MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); - } else if(isProp("pszMSG")) { - MsgSetMSG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(isProp("offAfterPRI")) { + pThis->offAfterPRI = pProp->val.num; + */ } else if(isProp("pszUxTradMsg")) { - MsgSetUxTradMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { - MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetTAG(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /* we need to create a property */ + CHKiRet(prop.Construct(&myProp)); + CHKiRet(prop.SetString(myProp, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr))); + CHKiRet(prop.ConstructFinalize(myProp)); + MsgSetInputName(pThis, myProp); + prop.Destruct(&myProp); } else if(isProp("pszRcvFromIP")) { - MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFromIPStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr), &propRcvFromIP); + prop.Destruct(&propRcvFromIP); } else if(isProp("pszRcvFrom")) { - MsgSetRcvFrom(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFromStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr), &propRcvFrom); + prop.Destruct(&propRcvFrom); } else if(isProp("pszHOSTNAME")) { - MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pCSStrucData")) { MsgSetStructuredData(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSAPPNAME")) { @@ -2514,8 +3109,11 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) memcpy(&pThis->tRcvdAt, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); } else if(isProp("tTIMESTAMP")) { memcpy(&pThis->tTIMESTAMP, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); + } else if(isProp("pszMSG")) { + dbgprintf("no longer supported property pszMSG silently ignored\n"); } +finalize_it: RETiRet; } #undef isProp @@ -2559,6 +3157,7 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_SERIALIZE, MsgSerialize); diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..b4b6d9f8 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -3,7 +3,7 @@ * * File begun on 2007-07-13 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -33,6 +33,7 @@ #include "syslogd-types.h" #include "template.h" + /* rgerhards 2004-11-08: The following structure represents a * syslog message. * @@ -47,59 +48,53 @@ * will be decremented. If it is 1, however, the object is actually * destroyed. To make this work, it is vital that MsgAddRef() is * called each time a "copy" is stored somewhere. + * + * WARNING: this structure is not calloc()ed, so be careful when + * adding new fields. You need to initialize them in + * msgBaseConstruct(). That function header comment also describes + * why this is the case. */ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - pthread_mutexattr_t mutAttr; -short bDoLock; /* use the mutex? */ - pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ + pthread_mutex_t mut; + sbool bDoLock; /* use the mutex? */ short iRefCount; /* reference counter (0 = unused) */ - short bParseHOSTNAME; /* should the hostname be parsed from the message? */ - /* background: the hostname is not present on "regular" messages - * received via UNIX domain sockets from the same machine. However, - * it is available when we have a forwarder (e.g. rfc3195d) using local - * sockets. All in all, the parser would need parse templates, that would - * resolve all these issues... rgerhards, 2005-10-06 - */ short iSeverity; /* the severity 0..7 */ - uchar *pszSeverity; /* severity as string... */ - int iLenSeverity; /* ... and its length. */ - uchar *pszSeverityStr; /* severity name... */ - int iLenSeverityStr; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ - uchar *pszFacility; /* Facility as string... */ - int iLenFacility; /* ... and its length. */ - uchar *pszFacilityStr; /* facility name... */ - int iLenFacilityStr; /* ... and its length. */ - uchar *pszPRI; /* the PRI as a string */ - int iLenPRI; /* and its length */ - uchar *pszRawMsg; /* message as it was received on the - * wire. This is important in case we - * need to preserve cryptographic verifiers. - */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ + short offMSG; /* offset at which the MSG part starts in pszRawMsg */ + short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + int msgFlags; /* flags associated with this message */ int iLenRawMsg; /* length of raw message */ - uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszUxTradMsg; /* the traditional UNIX message */ - int iLenUxTradMsg;/* Length of the traditional UNIX message */ - uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ - uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ int iLenHOSTNAME; /* Length of HOSTNAME */ - uchar *pszRcvFrom; /* System message was received from */ - int iLenRcvFrom; /* Length of pszRcvFrom */ - uchar *pszRcvFromIP; /* IP of system message was received from */ - int iLenRcvFromIP; /* Length of pszRcvFromIP */ - uchar *pszInputName; /* name of the input module that submitted this message */ - int iLenInputName; /* Length of pszInputName */ - short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we + * need to preserve cryptographic verifiers. */ + uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ + char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ + char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ + char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ + char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ + char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ + char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ + char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ + char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ cstr_t *pCSProgName; /* the (BSD) program name */ cstr_t *pCSStrucData; /* STRUCTURED-DATA */ cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + prop_t *pInputName; /* input name property */ + prop_t *pRcvFromIP; /* IP of system message was received from */ + union { + prop_t *pRcvFrom;/* name of system message was received from */ + struct sockaddr_storage *pfrominet; /* unresolved name */ + } rcvFrom; + + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp is used at later processing stages (namely in the output arena). Thanks to @@ -108,18 +103,18 @@ short bDoLock; /* use the mutex? */ enough to reliable, but I prefer to leave the subtle things to the OS, where it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ - char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ - char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ - char *pszRcvdAt_SecFrac;/* time just as fractional seconds (6 charcters) */ - char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ - char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ - char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ - char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ - char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ - char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ - char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ - int msgFlags; /* flags associated with this message */ + /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ + uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE]; + union { + uchar *pszTAG; /* pointer to tag value */ + uchar szBuf[CONF_TAG_BUFSIZE]; + } TAG; + char pszTimestamp3164[16]; + char pszTimestamp3339[33]; + char pszTIMESTAMP_SecFrac[7]; /* Note: a pointer is 64 bits/8 char, so this is actually fewer than a pointer! */ + char pszRcvdAt_SecFrac[7]; /* same as above. Both are fractional seconds for their respective timestamp */ }; @@ -132,64 +127,61 @@ short bDoLock; /* use the mutex? */ #define MARK 0x008 /* this message is a mark */ #define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ #define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ +#define NEEDS_DNSRESOL 0x040 /* fromhost address is unresolved and must be locked up via DNS reverse lookup first */ +#define NEEDS_ACLCHK_U 0x080 /* check UDP ACLs after DNS resolution has been done in main queue consumer */ /* function prototypes */ PROTOTYPEObjClassInit(msg); -char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -int getProtocolVersion(msg_t *pM); -char *getProtocolVersionString(msg_t *pM); -int getMSGLen(msg_t *pM); -char *getRawMsg(msg_t *pM); -char *getUxTradMsg(msg_t *pM); -char *getMSG(msg_t *pM); -char *getPRI(msg_t *pM); -int getPRIi(msg_t *pM); -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt); -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt); -char *getSeverity(msg_t *pM); -char *getSeverityStr(msg_t *pM); -char *getFacility(msg_t *pM); -char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, uchar*); +void MsgSetInputName(msg_t *pMsg, prop_t*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); -char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); -int getPROCIDLen(msg_t *pM); -char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); -void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); -void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); +void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); -char *getTAG(msg_t *pM); -int getHOSTNAMELen(msg_t *pM); -char *getHOSTNAME(msg_t *pM); -uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -char *getStructuredData(msg_t *pM); -int getProgramNameLen(msg_t *pM); -char *getProgramName(msg_t *pM); -void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); -rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); -void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); -void MsgSetMSG(msg_t *pMsg, char* pszMSG); -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); -void moveHOSTNAMEtoTAG(msg_t *pM); -char *getMSGID(msg_t *pM); -char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed); +rsRetVal msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa); +void MsgSetRcvFrom(msg_t *pMsg, prop_t*); +void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **); +rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp); +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); +rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); +void MsgSetMSGoffs(msg_t *pMsg, short offs); +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); +rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); +uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, + propid_t propID, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +uchar *getRcvFrom(msg_t *pM); + + +/* TODO: remove these five (so far used in action.c) */ +uchar *getMSG(msg_t *pM); +char *getHOSTNAME(msg_t *pM); +char *getPROCID(msg_t *pM, sbool bLockMutex); +char *getAPPNAME(msg_t *pM, sbool bLockMutex); +int getMSGLen(msg_t *pM); + +char *getHOSTNAME(msg_t *pM); +int getHOSTNAMELen(msg_t *pM); +uchar *getProgramName(msg_t *pM, sbool bLockMutex); +int getProgramNameLen(msg_t *pM, sbool bLockMutex); +uchar *getRcvFrom(msg_t *pM); +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); +uchar *propIDToName(propid_t propID); + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains @@ -200,6 +192,33 @@ rsRetVal MsgEnableThreadSafety(void); extern void (*funcMsgPrepareEnqueue)(msg_t *pMsg); #define MsgPrepareEnqueue(pMsg) funcMsgPrepareEnqueue(pMsg) + +/* ------------------------------ some inline functions ------------------------------ */ + +/* set raw message size. This is needed in some cases where a trunctation is necessary + * but the raw message must not be newly set. The most important (and currently only) + * use case is if we remove trailing LF or NUL characters. Note that the size can NOT + * be extended, only shrunk! + * rgerhards, 2009-08-26 + */ +static inline void +MsgSetRawMsgSize(msg_t *pMsg, size_t newLen) +{ + assert(newLen <= (size_t) pMsg->iLenRawMsg); + pMsg->iLenRawMsg = newLen; +} + + +/* get the ruleset that is associated with the ruleset. + * May be NULL. -- rgerhards, 2009-10-27 + */ +static inline ruleset_t* +MsgGetRuleset(msg_t *pMsg) +{ + return pMsg->pRuleset; +} + + #endif /* #ifndef MSG_H_INCLUDED */ /* vim:set ai: */ diff --git a/runtime/net.c b/runtime/net.c index db2d7e37..ab431f7c 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -173,7 +173,7 @@ AddPermittedPeerWildcard(permittedPeers_t *pPeer, uchar* pszStr, size_t lenStr) /* alloc memory for the domain component. We may waste a byte or * two, but that's ok. */ - CHKmalloc(pNew->pszDomainPart = malloc(lenStr +1 )); + CHKmalloc(pNew->pszDomainPart = MALLOC(lenStr +1 )); } if(pszStr[0] == '*') { @@ -695,7 +695,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS case AF_INET: /* add IPv4 */ iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -710,7 +710,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(sizeof(struct sockaddr_in))) + if((allowIP.addr.NetAddr = MALLOC(sizeof(struct sockaddr_in))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -732,7 +732,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 128; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -892,15 +892,18 @@ rsRetVal addAllowedSenderLine(char* pName, uchar** ppRestOfConfLine) * including IPv4/v6 as well as domain name wildcards. * This is a helper to isAllowedSender. As it is only called once, it is * declared inline. - * Returns 0 if they do not match, something else otherwise. - * contributed 1007-07-16 by mildew@gmail.com + * Returns 0 if they do not match, 1 if they match and 2 if a DNS name would have been required. + * contributed 2007-07-16 by mildew@gmail.com */ -static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost) +static inline int +MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { assert(pAllow != NULL); assert(pFrom != NULL); if(F_ISSET(pAllow->flags, ADDR_NAME)) { + if(bChkDNS == 0) + return 2; dbgprintf("MaskCmp: host=\"%s\"; pattern=\"%s\"\n", pszFromHost, pAllow->addr.HostWildcard); # if !defined(FNM_CASEFOLD) @@ -967,18 +970,22 @@ static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr /* check if a sender is allowed. The root of the the allowed sender. * list must be proveded by the caller. As such, this function can be * used to check both UDP and TCP allowed sender lists. - * returns 1, if the sender is allowed, 0 otherwise. + * returns 1, if the sender is allowed, 0 if not and 2 if we could not + * obtain a result because we would need a dns name, which we don't have + * (2 was added rgerhards, 2009-11-16). * rgerhards, 2005-09-26 */ -static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) +static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { struct AllowedSenders *pAllow; struct AllowedSenders *pAllowRoot; + int bNeededDNS = 0; /* partial check because we could not resolve DNS? */ + int ret; assert(pFrom != NULL); if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK) - return 0; /* if something went wrong, we denie access - that's the better choice... */ + return 0; /* if something went wrong, we deny access - that's the better choice... */ if(pAllowRoot == NULL) return 1; /* checking disabled, everything is valid! */ @@ -990,10 +997,20 @@ static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *p * that the sender is disallowed. */ for(pAllow = pAllowRoot ; pAllow != NULL ; pAllow = pAllow->pNext) { - if (MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost)) + ret = MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost, bChkDNS); + if(ret == 1) return 1; + else if(ret == 2) + bNeededDNS = 2; } - return 0; + return bNeededDNS; +} + + +/* legacy API, not to be used any longer */ +static int +isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) { + return isAllowedSender2(pszType, pFrom, pszFromHost, 1); } @@ -1306,7 +1323,7 @@ getLocalHostname(uchar **ppName) do { if(buf == NULL) { buf_len = 128; /* Initial guess */ - CHKmalloc(buf = malloc(buf_len)); + CHKmalloc(buf = MALLOC(buf_len)); } else { buf_len += buf_len; CHKmalloc(buf = realloc (buf, buf_len)); @@ -1370,7 +1387,7 @@ int *create_udp_socket(uchar *hostname, uchar *pszPort, int bIsServer) /* Count max number of sockets we may open */ for (maxs = 0, r = res; r != NULL ; r = r->ai_next, maxs++) /* EMPTY */; - socks = malloc((maxs+1) * sizeof(int)); + socks = MALLOC((maxs+1) * sizeof(int)); if (socks == NULL) { errmsg.LogError(0, NO_ERRCODE, "couldn't allocate memory for UDP sockets, suspending UDP message reception"); freeaddrinfo(res); @@ -1496,6 +1513,73 @@ int *create_udp_socket(uchar *hostname, uchar *pszPort, int bIsServer) } +/* check if two provided socket addresses point to the same host. Note that the + * length of the sockets must be provided as third parameter. This is necessary to + * compare non IPv4/v6 hosts, in which case we do a simple memory compare of the + * address structure (in that case, the same host may not reliably be detected). + * Note that we need to do the comparison not on the full structure, because it contains things + * like the port, which we do not need to look at when thinking about hostnames. So we look + * at the relevant fields, what means a somewhat more complicated processing. + * Also note that we use a non-standard calling interface, as this is much more natural and + * it looks extremely unlikely that we get an exception of any kind here. What we + * return is mimiced after memcmp(), and as such useful for building binary trees + * (the order relation may be a bit arbritrary, but at least it is consistent). + * rgerhards, 2009-09-03 + */ +static int CmpHost(struct sockaddr_storage *s1, struct sockaddr_storage* s2, size_t socklen) +{ + int ret; + + if(((struct sockaddr*) s1)->sa_family != ((struct sockaddr*) s2)->sa_family) { + ret = memcmp(s1, s2, socklen); + goto finalize_it; + } + + if(((struct sockaddr*) s1)->sa_family == AF_INET) { + if(((struct sockaddr_in *) s1)->sin_addr.s_addr == ((struct sockaddr_in*)s2)->sin_addr.s_addr) { + ret = 0; + } else if(((struct sockaddr_in *) s1)->sin_addr.s_addr < ((struct sockaddr_in*)s2)->sin_addr.s_addr) { + ret = -1; + } else { + ret = 1; + } + } else if(((struct sockaddr*) s1)->sa_family == AF_INET6) { + /* IPv6 addresses are always 16 octets long */ + ret = memcmp(((struct sockaddr_in6 *)s1)->sin6_addr.s6_addr, ((struct sockaddr_in6*)s2)->sin6_addr.s6_addr, 16); + } else { + ret = memcmp(s1, s2, socklen); + } + +finalize_it: + return ret; +} + + + +/* check if restrictions (ALCs) exists. The goal of this function is to disable the + * somewhat time-consuming ACL checks if no restrictions are defined (the usual case). + * This also permits to gain some speedup by using firewall-based ACLs instead of + * rsyslog ACLs (the recommended method. + * rgerhards, 2009-11-16 + */ +static rsRetVal +HasRestrictions(uchar *pszType, int *bHasRestrictions) { + struct AllowedSenders *pAllowRoot; + DEFiRet; + + CHKiRet(setAllowRoot(&pAllowRoot, pszType)); + + *bHasRestrictions = (pAllowRoot == NULL) ? 0 : 1; + +finalize_it: + if(iRet != RS_RET_OK) { + *bHasRestrictions = 1; /* in this case it is better to check individually */ + DBGPRINTF("Error %d trying to obtain ACL restriction state of '%s'\n", iRet, pszType); + } + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-03-05 */ @@ -1519,11 +1603,17 @@ CODESTARTobjQueryInterface(net) pIf->create_udp_socket = create_udp_socket; pIf->closeUDPListenSockets = closeUDPListenSockets; pIf->isAllowedSender = isAllowedSender; + pIf->isAllowedSender2 = isAllowedSender2; pIf->should_use_so_bsdcompat = should_use_so_bsdcompat; pIf->getLocalHostname = getLocalHostname; pIf->AddPermittedPeer = AddPermittedPeer; pIf->DestructPermittedPeers = DestructPermittedPeers; pIf->PermittedPeerWildcardMatch = PermittedPeerWildcardMatch; + pIf->CmpHost = CmpHost; + pIf->HasRestrictions = HasRestrictions; + /* data members */ + pIf->pACLAddHostnameOnFail = &ACLAddHostnameOnFail; + pIf->pACLDontResolve = &ACLDontResolve; finalize_it: ENDobjQueryInterface(net) diff --git a/runtime/net.h b/runtime/net.h index 092c3116..101ce79d 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -139,18 +139,23 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ void (*debugListenInfo)(int fd, char *type); int *(*create_udp_socket)(uchar *hostname, uchar *LogPort, int bIsServer); void (*closeUDPListenSockets)(int *finet); - int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); + int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); /* deprecated! */ rsRetVal (*getLocalHostname)(uchar**); int (*should_use_so_bsdcompat)(void); /* permitted peer handling should be replaced by something better (see comments above) */ rsRetVal (*AddPermittedPeer)(permittedPeers_t **ppRootPeer, uchar *pszID); rsRetVal (*DestructPermittedPeers)(permittedPeers_t **ppRootPeer); rsRetVal (*PermittedPeerWildcardMatch)(permittedPeers_t *pPeer, uchar *pszNameToMatch, int *pbIsMatching); + /* v5 interface additions */ + int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t); + /* v6 interface additions - 2009-11-16 */ + rsRetVal (*HasRestrictions)(uchar *, int *bHasRestrictions); + int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS); /* data members - these should go away over time... TODO */ int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ ENDinterface(net) -#define netCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +#define netCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* prototypes */ PROTOTYPEObj(net); diff --git a/runtime/netstrm.c b/runtime/netstrm.c index f6a8de7f..3658006f 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -114,7 +114,6 @@ AcceptConnReq(netstrm_t *pThis, netstrm_t **ppNew) ISOBJ_TYPE_assert(pThis, netstrm); assert(ppNew != NULL); -RUNLOG_STR("XXX: accept conn reqeust"); /* accept the new connection */ CHKiRet(pThis->Drvr.AcceptConnReq(pThis->pDrvrData, &pNewNsd)); /* construct our object so that we can use it... */ @@ -234,6 +233,19 @@ Send(netstrm_t *pThis, uchar *pBuf, ssize_t *pLenBuf) RETiRet; } +/* Enable Keep-Alive handling for those drivers that support it. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(netstrm_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, netstrm); + iRet = pThis->Drvr.EnableKeepAlive(pThis->pDrvrData); + RETiRet; +} + + /* check connection - slim wrapper for NSD driver function */ static void @@ -338,6 +350,7 @@ CODESTARTobjQueryInterface(netstrm) pIf->SetDrvrPermPeers = SetDrvrPermPeers; pIf->CheckConnection = CheckConnection; pIf->GetSock = GetSock; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(netstrm) diff --git a/runtime/netstrm.h b/runtime/netstrm.h index b00dd223..f6931104 100644 --- a/runtime/netstrm.h +++ b/runtime/netstrm.h @@ -69,9 +69,13 @@ BEGINinterface(netstrm) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v4 */ + rsRetVal (*EnableKeepAlive)(netstrm_t *pThis); ENDinterface(netstrm) -#define netstrmCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ -/* interface version 3 added GetRemAddr() */ +#define netstrmCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +/* interface version 3 added GetRemAddr() + * interface version 4 added EnableKeepAlive() -- rgerhards, 2009-06-02 + * */ /* prototypes */ PROTOTYPEObj(netstrm); diff --git a/runtime/netstrms.c b/runtime/netstrms.c index 6b28e7ea..e9ff2568 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -36,6 +36,7 @@ #include "nsd.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "netstrms.h" MODULE_TYPE_LIB @@ -304,6 +305,7 @@ ENDObjClassInit(netstrms) BEGINmodExit CODESTARTmodExit nsselClassExit(); + nspollClassExit(); netstrmsClassExit(); netstrmClassExit(); /* we use this object, so we must exit it after we are finished */ ENDmodExit @@ -322,6 +324,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(netstrmClassInit(pModInfo)); CHKiRet(nsselClassInit(pModInfo)); + CHKiRet(nspollClassInit(pModInfo)); CHKiRet(netstrmsClassInit(pModInfo)); ENDmodInit /* vi:set ai: diff --git a/runtime/nsd.h b/runtime/nsd.h index f0c9b9b6..e5b9320b 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -69,9 +69,13 @@ BEGINinterface(nsd) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v5 */ + rsRetVal (*EnableKeepAlive)(nsd_t *pThis); ENDinterface(nsd) -#define nsdCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ -/* interface version 4 added GetRemAddr() */ +#define nsdCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +/* interface version 4 added GetRemAddr() + * interface version 5 added EnableKeepAlive() -- rgerhards, 2009-06-02 + */ /* interface for the select call */ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ @@ -83,4 +87,13 @@ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ ENDinterface(nsdsel) #define nsdselCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface for the epoll call */ +BEGINinterface(nsdpoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nsdpoll_t **ppThis); + rsRetVal (*Destruct)(nsdpoll_t **ppThis); + rsRetVal (*Ctl)(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op); + rsRetVal (*Wait)(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); +ENDinterface(nsdpoll) +#define nsdpollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + #endif /* #ifndef INCLUDED_NSD_H */ diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 5786e191..0ee70e56 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -44,6 +44,7 @@ #include "stringbuf.h" #include "errmsg.h" #include "net.h" +#include "datetime.h" #include "nsd_ptcp.h" #include "nsdsel_gtls.h" #include "nsd_gtls.h" @@ -61,6 +62,7 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) +DEFobjCurrIf(datetime) DEFobjCurrIf(nsd_ptcp) static int bGlblSrvrInitDone = 0; /**< 0 - server global init not yet done, 1 - already done */ @@ -129,7 +131,7 @@ readFile(uchar *pszFile, gnutls_datum_t *pBuf) ABORT_FINALIZE(RS_RET_FILE_TOO_LARGE); } - CHKmalloc(pBuf->data = malloc(stat_st.st_size)); + CHKmalloc(pBuf->data = MALLOC(stat_st.st_size)); pBuf->size = stat_st.st_size; if(read(fd, pBuf->data, stat_st.st_size) != stat_st.st_size) { errmsg.LogError(0, RS_RET_IO_ERROR, "error or incomplete read of file '%s'", pszFile); @@ -335,7 +337,7 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) gnutls_x509_crt_deinit(cert); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; finalize_it: @@ -455,7 +457,7 @@ GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) snprintf((char*)buf, sizeof(buf), ":%2.2X", pFingerprint[i]); CHKiRet(rsCStrAppendStrWithLen(pStr, buf, 3)); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -710,20 +712,20 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) } /* we found a common name, now extract it */ - CHKiRet(rsCStrConstruct(&pstrCN)); + CHKiRet(cstrConstruct(&pstrCN)); while(szDN[i] != '\0' && szDN[i] != ',') { if(szDN[i] == '\\') { /* hex escapes are not implemented */ ++i; /* escape char processed */ if(szDN[i] == '\0') ABORT_FINALIZE(RS_RET_CERT_INVALID_DN); - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } else { - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } ++i; /* char processed */ } - CHKiRet(rsCStrFinish(pstrCN)); + CHKiRet(cstrFinalize(pstrCN)); /* we got it - we ignore the rest of the DN string (if any). So we may * not detect if it contains more than one CN @@ -734,7 +736,7 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) finalize_it: if(iRet != RS_RET_OK) { if(pstrCN != NULL) - rsCStrDestruct(&pstrCN); + cstrDestruct(&pstrCN); } RETiRet; @@ -761,7 +763,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) size = sizeof(fingerprint); CHKgnutls(gnutls_x509_crt_get_fingerprint(*pCert, GNUTLS_DIG_SHA1, fingerprint, &size)); CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint)); - dbgprintf("peer's certificate SHA1 fingerprint: %s\n", rsCStrGetSzStr(pstrFingerprint)); + dbgprintf("peer's certificate SHA1 fingerprint: %s\n", cstrGetSzStr(pstrFingerprint)); /* now search through the permitted peers to see if we can find a permitted one */ bFoundPositiveMatch = 0; @@ -779,7 +781,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) if(pThis->bReportAuthErr == 1) { errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer fingerprint '%s' unknown - we are " - "not permitted to talk to it", rsCStrGetSzStr(pstrFingerprint)); + "not permitted to talk to it", cstrGetSzStr(pstrFingerprint)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -787,7 +789,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) finalize_it: if(pstrFingerprint != NULL) - rsCStrDestruct(&pstrFingerprint); + cstrDestruct(&pstrFingerprint); RETiRet; } @@ -874,21 +876,21 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) /* if we did not succeed so far, we try the CN part of the DN... */ CHKiRet(gtlsGetCN(pThis, pCert, &pstrCN)); if(pstrCN != NULL) { /* NULL if there was no CN present */ - dbgprintf("gtls now checking auth for CN '%s'\n", rsCStrGetSzStr(pstrCN)); - snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", rsCStrGetSzStr(pstrCN)); + dbgprintf("gtls now checking auth for CN '%s'\n", cstrGetSzStr(pstrCN)); + snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", cstrGetSzStr(pstrCN)); CHKiRet(rsCStrAppendStr(pStr, lnBuf)); - CHKiRet(gtlsChkOnePeerName(pThis, rsCStrGetSzStr(pstrCN), &bFoundPositiveMatch)); + CHKiRet(gtlsChkOnePeerName(pThis, cstrGetSzStr(pstrCN), &bFoundPositiveMatch)); } } if(!bFoundPositiveMatch) { dbgprintf("invalid peer name, not permitted to talk to it\n"); if(pThis->bReportAuthErr == 1) { - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer name not authorized - " "not permitted to talk to it. Names: %s", - rsCStrGetSzStr(pStr)); + cstrGetSzStr(pStr)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -1010,13 +1012,13 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) errmsg.LogError(0, NO_ERRCODE, "not permitted to talk to peer, certificate invalid: %s", pszErrCause); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_INVALID); } /* get current time for certificate validation */ - if(time(&ttNow) == -1) + if(datetime.GetTime(&ttNow) == -1) ABORT_FINALIZE(RS_RET_SYS_ERR); /* as it looks, we need to validate the expiration dates ourselves... @@ -1032,8 +1034,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert > ttNow) { errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "not permitted to talk to peer: certificate %d not yet active", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_NOT_YET_ACTIVE); } @@ -1043,8 +1045,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert < ttNow) { errmsg.LogError(0, RS_RET_CERT_EXPIRED, "not permitted to talk to peer: certificate %d expired", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_EXPIRED); } gnutls_x509_crt_deinit(cert); @@ -1325,7 +1327,10 @@ finalize_it: static void CheckConnection(nsd_t __attribute__((unused)) *pNsd) { - /* dummy, do nothing */ + nsd_gtls_t *pThis = (nsd_gtls_t*) pNsd; + ISOBJ_TYPE_assert(pThis, nsd_gtls); + + nsd_ptcp.CheckConnection(pThis->pTcp); } @@ -1479,7 +1484,7 @@ Rcv(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) if(pThis->pszRcvBuf == NULL) { /* we have no buffer, so we need to malloc one */ - CHKmalloc(pThis->pszRcvBuf = malloc(NSD_GTLS_MAX_RCVBUF)); + CHKmalloc(pThis->pszRcvBuf = MALLOC(NSD_GTLS_MAX_RCVBUF)); pThis->lenRcvBuf = -1; } @@ -1560,6 +1565,16 @@ finalize_it: RETiRet; } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + return nsd_ptcp.EnableKeepAlive(pNsd); +} + + /* open a connection to a remote host (server). With GnuTLS, we always * open a plain tcp socket and then, if in TLS mode, do a handshake on it. @@ -1669,6 +1684,7 @@ CODESTARTobjQueryInterface(nsd_gtls) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->GetRemAddr = GetRemAddr; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_gtls) @@ -1683,6 +1699,7 @@ CODESTARTObjClassExit(nsd_gtls) objRelease(nsd_ptcp, LM_NSD_PTCP_FILENAME); objRelease(net, LM_NET_FILENAME); objRelease(glbl, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); ENDObjClassExit(nsd_gtls) @@ -1694,6 +1711,7 @@ ENDObjClassExit(nsd_gtls) BEGINObjClassInit(nsd_gtls, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); CHKiRet(objUse(nsd_ptcp, LM_NSD_PTCP_FILENAME)); diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index cc531ca0..744955c7 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -48,6 +48,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nsdsel_ptcp.h" +#include "nsdpoll_ptcp.h" #include "nsd_ptcp.h" MODULE_TYPE_LIB @@ -296,12 +297,12 @@ FillRemHost(nsd_ptcp_t *pThis, struct sockaddr *pAddr) * memory consumption) */ len = strlen((char*)szIP) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostIP = malloc(len)) == NULL) + if((pThis->pRemHostIP = MALLOC(len)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pRemHostIP, szIP, len); len = strlen((char*)szHname) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostName = malloc(len)) == NULL) { + if((pThis->pRemHostName = MALLOC(len)) == NULL) { free(pThis->pRemHostIP); /* prevent leak */ pThis->pRemHostIP = NULL; ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -562,6 +563,7 @@ finalize_it: static rsRetVal Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) { + char errStr[1024]; DEFiRet; nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; ISOBJ_TYPE_assert(pThis, nsd_ptcp); @@ -571,7 +573,9 @@ Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) if(*pLenBuf == 0) { ABORT_FINALIZE(RS_RET_CLOSED); } else if (*pLenBuf < 0) { - ABORT_FINALIZE(RS_RET_ERR); + rs_strerror_r(errno, errStr, sizeof(errStr)); + dbgprintf("error during recv on NSD %p: %s\n", pNsd, errStr); + ABORT_FINALIZE(RS_RET_RCV_ERR); } finalize_it: @@ -614,6 +618,34 @@ finalize_it: } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; + int ret; + int optval; + socklen_t optlen; + DEFiRet; + ISOBJ_TYPE_assert(pThis, nsd_ptcp); + + optval = 1; + optlen = sizeof(optval); + ret = setsockopt(pThis->sock, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen); + if(ret < 0) { + dbgprintf("EnableKeepAlive socket call returns error %d\n", ret); + ABORT_FINALIZE(RS_RET_ERR); + } + + dbgprintf("KEEPALIVE enabled for nsd %p\n", pThis); + +finalize_it: + RETiRet; +} + + /* open a connection to a remote host (server). * rgerhards, 2008-03-19 */ @@ -754,6 +786,7 @@ CODESTARTobjQueryInterface(nsd_ptcp) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->CheckConnection = CheckConnection; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_ptcp) @@ -792,6 +825,7 @@ ENDObjClassInit(nsd_ptcp) BEGINmodExit CODESTARTmodExit + nsdpoll_ptcpClassExit(); nsdsel_ptcpClassExit(); nsd_ptcpClassExit(); ENDmodExit @@ -810,6 +844,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(nsd_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ CHKiRet(nsdsel_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + CHKiRet(nsdpoll_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ ENDmodInit /* vi:set ai: */ diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c new file mode 100644 index 00000000..51006707 --- /dev/null +++ b/runtime/nsdpoll_ptcp.c @@ -0,0 +1,288 @@ +/* nsdpoll_ptcp.c + * + * An implementation of the nsd epoll() interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#ifdef HAVE_EPOLL_CREATE /* this module requires epoll! */ + +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "errmsg.h" +#include "srUtils.h" +#include "nspoll.h" +#include "nsd_ptcp.h" +#include "nsdpoll_ptcp.h" +#include "unlimited_select.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + + +/* -START------------------------- helpers for event list ------------------------------------ */ + +/* add new entry to list. We assume that the fd is not already present and DO NOT check this! + * Returns newly created entry in pEvtLst. + * Note that we currently need to use level-triggered mode, because the upper layers do not work + * in parallel. As such, in edge-triggered mode we may not get notified, because new data comes + * in after we have read everything that was present. To use ET mode, we need to change the upper + * peers so that they immediately start a new wait before processing the data read. That obviously + * requires more elaborate redesign and we postpone this until the current more simplictic mode has + * been proven OK in practice. + * rgerhards, 2009-11-18 + */ +static inline rsRetVal +addEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, int mode, nsd_ptcp_t *pSock, nsdpoll_epollevt_lst_t **pEvtLst) { + nsdpoll_epollevt_lst_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (nsdpoll_epollevt_lst_t*) malloc(sizeof(nsdpoll_epollevt_lst_t))); + pNew->id = id; + pNew->pUsr = pUsr; + pNew->pSock = pSock; + pNew->event.events = 0; /* TODO: at some time we should be able to use EPOLLET */ + if(mode & NSDPOLL_IN) + pNew->event.events |= EPOLLIN; + if(mode & NSDPOLL_OUT) + pNew->event.events |= EPOLLOUT; + pNew->event.data.u64 = (uint64) pNew; + pNew->pNext = pThis->pRoot; + pThis->pRoot = pNew; + *pEvtLst = pNew; + +finalize_it: + RETiRet; +} + + +/* find and unlink the entry identified by id/pUsr from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +unlinkEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, nsdpoll_epollevt_lst_t **ppEvtLst) { + nsdpoll_epollevt_lst_t *pEvtLst; + nsdpoll_epollevt_lst_t *pPrev = NULL; + DEFiRet; + + pEvtLst = pThis->pRoot; + while(pEvtLst != NULL && !(pEvtLst->id == id && pEvtLst->pUsr == pUsr)) { + pPrev = pEvtLst; + pEvtLst = pEvtLst->pNext; + } + if(pEvtLst == NULL) + ABORT_FINALIZE(RS_RET_NOT_FOUND); + + *ppEvtLst = pEvtLst; + + /* unlink */ + if(pPrev == NULL) + pThis->pRoot = pEvtLst->pNext; + else + pPrev->pNext = pEvtLst->pNext; + +finalize_it: + RETiRet; +} + + +/* destruct the provided element. It must already be unlinked from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) { + DEFiRet; + free(*ppEvtLst); + *ppEvtLst = NULL; + RETiRet; +} + + +/* -END--------------------------- helpers for event list ------------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */ +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); + pThis->efd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n"); + pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */ +# endif + if(pThis->efd < 0) { + DBGPRINTF("epoll_create1() could not create fd\n"); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } +finalize_it: +ENDobjConstruct(nsdpoll_ptcp) + + +/* destructor for the nsdpoll_ptcp object */ +BEGINobjDestruct(nsdpoll_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nsdpoll_ptcp) +ENDobjDestruct(nsdpoll_ptcp) + + +/* Modify socket set */ +static rsRetVal +Ctl(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; + nsdpoll_epollevt_lst_t *pEventLst; + int errSave; + char errStr[512]; + DEFiRet; + + if(op == NSDPOLL_ADD) { + dbgprintf("adding nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(addEvent(pThis, id, pUsr, mode, pSock, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_ADD, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + } + } else if(op == NSDPOLL_DEL) { + dbgprintf("removing nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(unlinkEvent(pThis, id, pUsr, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_DEL, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + ABORT_FINALIZE(RS_RET_ERR_EPOLL_CTL); + } + CHKiRet(delEvent(&pEventLst)); + } else { + dbgprintf("program error: invalid NSDPOLL_mode %d - ignoring request\n", op); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Wait for io to become ready. After the successful call, idRdy contains the + * id set by the caller for that i/o event, ppUsr is a pointer to a location + * where the user pointer shall be stored. + * TODO: this is a trivial implementation that only polls one event at a time. We + * may later extend it to poll for multiple events, what would cause less + * overhead. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Wait(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsdpoll_epollevt_lst_t *pOurEvt; + struct epoll_event event; + int nfds; + DEFiRet; + + assert(idRdy != NULL); + assert(ppUsr != NULL); + + nfds = epoll_wait(pThis->efd, &event, 1, timeout); + if(nfds == -1) { + if(errno == EINTR) { + ABORT_FINALIZE(RS_RET_EINTR); + } else { + DBGPRINTF("epoll() returned with error code %d\n", errno); + ABORT_FINALIZE(RS_RET_ERR_EPOLL); + } + } else if(nfds == 0) { + ABORT_FINALIZE(RS_RET_TIMEOUT); + } + + /* we got a valid event, so tell the caller... */ + pOurEvt = (nsdpoll_epollevt_lst_t*) event.data.u64; + *idRdy = pOurEvt->id; + *ppUsr = pOurEvt->pUsr; + +finalize_it: + RETiRet; +} + + +/* ------------------------------ end support for the epoll() interface ------------------------------ */ + + +/* queryInterface function */ +BEGINobjQueryInterface(nsdpoll_ptcp) +CODESTARTobjQueryInterface(nsdpoll_ptcp) + if(pIf->ifVersion != nsdCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpConstruct; + pIf->Destruct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpDestruct; + pIf->Ctl = Ctl; + pIf->Wait = Wait; +finalize_it: +ENDobjQueryInterface(nsdpoll_ptcp) + + +/* exit our class + */ +BEGINObjClassExit(nsdpoll_ptcp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdpoll_ptcp) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(nsdpoll_ptcp) + + +/* Initialize the nsdpoll_ptcp class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nsdpoll_ptcp, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nsdpoll_ptcp) +#endif /* #ifdef HAVE_EPOLL_CREATE this module requires epoll! */ + +/* vi:set ai: + */ diff --git a/runtime/nsdpoll_ptcp.h b/runtime/nsdpoll_ptcp.h new file mode 100644 index 00000000..cea2823d --- /dev/null +++ b/runtime/nsdpoll_ptcp.h @@ -0,0 +1,60 @@ +/* An implementation of the nsd poll interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSDPOLL_PTCP_H +#define INCLUDED_NSDPOLL_PTCP_H + +#include "nsd.h" +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif +typedef nsdpoll_if_t nsdpoll_ptcp_if_t; /* we just *implement* this interface */ +/* a helper object to keep track of the epoll event records + * Note that we need to keep track of that list because we need to + * free the events when they are no longer needed. + */ +typedef struct nsdpoll_epollevt_lst_s nsdpoll_epollevt_lst_t; +struct nsdpoll_epollevt_lst_s { +#if HAVE_SYS_EPOLL_H + epoll_event_t event; +#endif + int id; + void *pUsr; + nsd_ptcp_t *pSock; /* our associated netstream driver data */ + nsdpoll_epollevt_lst_t *pNext; +}; + +/* the nsdpoll_ptcp object */ +struct nsdpoll_ptcp_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int efd; /* file descriptor used by epoll */ + nsdpoll_epollevt_lst_t *pRoot; /* Root of the epoll event list */ +}; + +/* interface is defined in nsd.h, we just implement it! */ +#define nsdpoll_ptcpCURR_IF_VERSION nsdCURR_IF_VERSION + +/* prototypes */ +PROTOTYPEObj(nsdpoll_ptcp); + +#endif /* #ifndef INCLUDED_NSDPOLL_PTCP_H */ diff --git a/runtime/nsdsel_ptcp.c b/runtime/nsdsel_ptcp.c index 41b85e0c..e2cfca7c 100644 --- a/runtime/nsdsel_ptcp.c +++ b/runtime/nsdsel_ptcp.c @@ -36,6 +36,7 @@ #include "errmsg.h" #include "nsd_ptcp.h" #include "nsdsel_ptcp.h" +#include "unlimited_select.h" /* static data */ DEFobjStaticHelpers @@ -47,14 +48,23 @@ DEFobjCurrIf(glbl) */ BEGINobjConstruct(nsdsel_ptcp) /* be sure to specify the object type also in END macro! */ pThis->maxfds = 0; +#ifdef USE_UNLIMITED_SELECT + pThis->pReadfds = calloc(1, glbl.GetFdSetSize()); + pThis->pWritefds = calloc(1, glbl.GetFdSetSize()); +#else FD_ZERO(&pThis->readfds); FD_ZERO(&pThis->writefds); +#endif ENDobjConstruct(nsdsel_ptcp) /* destructor for the nsdsel_ptcp object */ BEGINobjDestruct(nsdsel_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(nsdsel_ptcp) +#ifdef USE_UNLIMITED_SELECT + freeFdSet(pThis->pReadfds); + freeFdSet(pThis->pWritefds); +#endif ENDobjDestruct(nsdsel_ptcp) @@ -65,20 +75,27 @@ Add(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pSock, nsd_ptcp); ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); switch(waitOp) { case NSDSEL_RD: - FD_SET(pSock->sock, &pThis->readfds); + FD_SET(pSock->sock, pReadfds); break; case NSDSEL_WR: - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - FD_SET(pSock->sock, &pThis->readfds); - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pReadfds); + FD_SET(pSock->sock, pWritefds); break; } @@ -98,6 +115,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) DEFiRet; int i; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); assert(piNumReady != NULL); @@ -106,13 +130,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) // TODO: name in dbgprintf! dbgprintf("--------<NSDSEL_PTCP> calling select, active fds (max %d): ", pThis->maxfds); for(i = 0; i <= pThis->maxfds; ++i) - if(FD_ISSET(i, &pThis->readfds) || FD_ISSET(i, &pThis->writefds)) + if(FD_ISSET(i, pReadfds) || FD_ISSET(i, pWritefds)) dbgprintf("%d ", i); dbgprintf("\n"); } /* now do the select */ - *piNumReady = select(pThis->maxfds+1, &pThis->readfds, &pThis->writefds, NULL, NULL); + *piNumReady = select(pThis->maxfds+1, pReadfds, pWritefds, NULL, NULL); RETiRet; } @@ -125,6 +149,13 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); ISOBJ_TYPE_assert(pSock, nsd_ptcp); @@ -132,14 +163,14 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) switch(waitOp) { case NSDSEL_RD: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds); break; case NSDSEL_WR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds) - | FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds) + | FD_ISSET(pSock->sock, pWritefds); break; } diff --git a/runtime/nsdsel_ptcp.h b/runtime/nsdsel_ptcp.h index 6c0c7fa7..f9ec8210 100644 --- a/runtime/nsdsel_ptcp.h +++ b/runtime/nsdsel_ptcp.h @@ -31,8 +31,13 @@ typedef nsdsel_if_t nsdsel_ptcp_if_t; /* we just *implement* this interface */ struct nsdsel_ptcp_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ int maxfds; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds; + fd_set *pWritefds; +#else fd_set readfds; fd_set writefds; +#endif }; /* interface is defined in nsd.h, we just implement it! */ diff --git a/runtime/nspoll.c b/runtime/nspoll.c new file mode 100644 index 00000000..f287cd4e --- /dev/null +++ b/runtime/nspoll.c @@ -0,0 +1,198 @@ +/* nspoll.c + * + * This is an io waiter interface utilizing the much-more-efficient poll/epoll API. + * Note that it may not always be available for a given driver. If so, that is reported + * back to the upper peer which then should consult a nssel-based io waiter. + * + * Work on this module begun 2009-11-18 by Rainer Gerhards. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include "rsyslog.h" +#include "obj.h" +#include "module-template.h" +#include "netstrm.h" +#include "nspoll.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) + + +/* load our low-level driver. This must be done before any + * driver-specific functions (allmost all...) can be carried + * out. Note that the driver's .ifIsLoaded is correctly + * initialized by calloc() and we depend on that. Please note that + * we do some name-mangeling. We know that each nsd driver also needs + * a nspoll driver. So we simply append "sel" to the nsd driver name: This, + * of course, means that the driver name must match these rules, but that + * shouldn't be a real problem. + * WARNING: this code is mostly identical to similar code in + * netstrms.c - TODO: abstract it and move it to some common place. + * rgerhards, 2008-04-28 + */ +static rsRetVal +loadDrvr(nspoll_t *pThis) +{ + DEFiRet; + uchar *pBaseDrvrName; + uchar szDrvrName[48]; /* 48 shall be large enough */ + + pBaseDrvrName = pThis->pBaseDrvrName; + if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdpoll_%s", pBaseDrvrName) == sizeof(szDrvrName)) + ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); + CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); + + pThis->Drvr.ifVersion = nsdCURR_IF_VERSION; + /* The pDrvrName+2 below is a hack to obtain the object name. It + * safes us to have yet another variable with the name without "lm" in + * front of it. If we change the module load interface, we may re-think + * about this hack, but for the time being it is efficient and clean + * enough. -- rgerhards, 2008-04-18 + */ +RUNLOG_VAR("%s", szDrvrName+2); + CHKiRet(obj.UseObj(__FILE__, szDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr)); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pDrvrName != NULL) + free(pThis->pDrvrName); + pThis->pDrvrName = NULL; + } + RETiRet; +} + + +/* Standard-Constructor */ +BEGINobjConstruct(nspoll) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(nspoll) + + +/* destructor for the nspoll object */ +BEGINobjDestruct(nspoll) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nspoll) + if(pThis->pDrvrData != NULL) + pThis->Drvr.Destruct(&pThis->pDrvrData); + + /* and now we must release our driver, if we got one. We use the presence of + * a driver name string as load indicator (because we also need that string + * to release the driver + */ + if(pThis->pDrvrName != NULL) { + obj.ReleaseObj(__FILE__, pThis->pDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr); + free(pThis->pDrvrName); + } +ENDobjDestruct(nspoll) + + +/* ConstructionFinalizer */ +static rsRetVal +ConstructFinalize(nspoll_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); +RUNLOG_STR("trying to load epoll driver\n"); + CHKiRet(loadDrvr(pThis)); + CHKiRet(pThis->Drvr.Construct(&pThis->pDrvrData)); +finalize_it: +dbgprintf("XXX: done trying to load epoll driver, state %d\n", iRet); + RETiRet; +} + + +/* Carries out the actual wait (all done in lower layers) + */ +static rsRetVal +Wait(nspoll_t *pThis, int timeout, int *idRdy, void **ppUsr) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + assert(idRdy != NULL); + iRet = pThis->Drvr.Wait(pThis->pDrvrData, timeout, idRdy, ppUsr); + RETiRet; +} + + +/* semantics like the epoll_ctl() function, does the same thing. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Ctl(nspoll_t *pThis, netstrm_t *pStrm, int id, void *pUsr, int mode, int op) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + iRet = pThis->Drvr.Ctl(pThis->pDrvrData, pStrm->pDrvrData, id, pUsr, mode, op); + RETiRet; +} + + +/* queryInterface function */ +BEGINobjQueryInterface(nspoll) +CODESTARTobjQueryInterface(nspoll) + if(pIf->ifVersion != nspollCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = nspollConstruct; + pIf->ConstructFinalize = ConstructFinalize; + pIf->Destruct = nspollDestruct; + pIf->Wait = Wait; + pIf->Ctl = Ctl; +finalize_it: +ENDobjQueryInterface(nspoll) + + +/* exit our class + */ +BEGINObjClassExit(nspoll, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nspoll) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(nspoll) + + +/* Initialize the nspoll class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nspoll, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + DBGPRINTF("doing nspollClassInit\n"); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nspoll) +/* vi:set ai: + */ diff --git a/runtime/nspoll.h b/runtime/nspoll.h new file mode 100644 index 00000000..a77759c0 --- /dev/null +++ b/runtime/nspoll.h @@ -0,0 +1,65 @@ +/* Definitions for the nspoll io activity waiter + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSPOLL_H +#define INCLUDED_NSPOLL_H + +#include "netstrms.h" + +/* some operations to be portable when we do not have epoll() available */ +#define NSDPOLL_ADD 1 +#define NSDPOLL_DEL 2 + +/* and some mode specifiers for waiting on input/output */ +#define NSDPOLL_IN 1 /* EPOLLIN */ +#define NSDPOLL_OUT 2 /* EPOLLOUT */ +/* next is 4, 8, 16, ... - must be bit values, as they are ored! */ + +/* the nspoll object */ +struct nspoll_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + nsd_t *pDrvrData; /**< the driver's data elements */ + uchar *pBaseDrvrName; /**< nsd base driver name to use, or NULL if system default */ + uchar *pDrvrName; /**< full base driver name (set when driver is loaded) */ + nsdpoll_if_t Drvr; /**< our stream driver */ +}; + + +/* interface */ +BEGINinterface(nspoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nspoll_t **ppThis); + rsRetVal (*ConstructFinalize)(nspoll_t *pThis); + rsRetVal (*Destruct)(nspoll_t **ppThis); + rsRetVal (*Wait)(nspoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); + rsRetVal (*Ctl)(nspoll_t *pNsdpoll, netstrm_t *pStrm, int id, void *pUsr, int mode, int op); + rsRetVal (*IsEPollSupported)(void); /* static method */ +ENDinterface(nspoll) +#define nspollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + +/* prototypes */ +PROTOTYPEObj(nspoll); + +/* the name of our library binary */ +#define LM_NSPOLL_FILENAME LM_NETSTRMS_FILENAME + +#endif /* #ifndef INCLUDED_NSPOLL_H */ diff --git a/runtime/nssel.c b/runtime/nssel.c index d11d5fe1..7c5be3a9 100644 --- a/runtime/nssel.c +++ b/runtime/nssel.c @@ -219,6 +219,7 @@ ENDObjClassExit(nssel) */ BEGINObjClassInit(nssel, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + DBGPRINTF("doing nsselClassInit\n"); CHKiRet(objUse(glbl, CORE_COMPONENT)); /* set our own handlers */ diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 914c2f2c..e1b54d4f 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -105,12 +105,13 @@ struct obj_s { /* the dummy struct that each derived class can be casted to */ # define ISOBJ_TYPE_assert(pObj, objType) \ do { \ ASSERT(pObj != NULL); \ - ASSERT((unsigned) ((obj_t*) (pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ if(strcmp((char*)(((obj_t*)pObj)->pObjInfo->pszID), #objType)) { \ dbgprintf("%s:%d ISOBJ assert failure: invalid object type, expected '%s' " \ - "actual '%s'\n", __FILE__, __LINE__, #objType, (((obj_t*)pObj)->pObjInfo->pszID)); \ + "actual '%s', cookie: %X\n", __FILE__, __LINE__, #objType, \ + (((obj_t*)pObj)->pObjInfo->pszID), ((obj_t*)(pObj))->iObjCooCKiE); \ assert(0); /* trigger assertion, messge we already have */ \ } \ + ASSERT((unsigned) ((obj_t*)(pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ } while(0) #else /* non-debug mode, no checks but much faster */ # define BEGINobjInstance obj_t objData @@ -280,7 +281,7 @@ rsRetVal objName##ClassExit(void) \ * rgerhards, 2008-01-30 */ #define BEGINobjDestruct(OBJ) \ - rsRetVal OBJ##Destruct(OBJ##_t **ppThis) \ + rsRetVal OBJ##Destruct(OBJ##_t __attribute__((unused)) **ppThis) \ { \ DEFiRet; \ int iCancelStateSave; \ @@ -292,6 +293,15 @@ rsRetVal objName##ClassExit(void) \ ISOBJ_TYPE_assert(pThis, OBJ); \ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); +/* note: there was a long-time bug in the macro below that lead to *ppThis = NULL + * only when the object was actually destructed. I discovered this issue during + * introduction of the pRcvFrom property in msg_t, but it potentially had other + * effects, too. I am not sure if some experienced instability resulted from this + * bug OR if its fix will cause harm to so-far "correctly" running code. The later + * may very well be. Thus I will change it only for the current branch and also + * the beta, but not in all old builds. Let's see how things evolve. + * rgerhards, 2009-06-30 + */ #define ENDobjDestruct(OBJ) \ goto finalize_it; /* prevent compiler warning ;) */ \ /* no more code here! */ \ @@ -299,8 +309,8 @@ rsRetVal objName##ClassExit(void) \ if(pThis != NULL) { \ obj.DestructObjSelf((obj_t*) pThis); \ free(pThis); \ - *ppThis = NULL; \ } \ + *ppThis = NULL; \ pthread_setcancelstate(iCancelStateSave, NULL); \ RETiRet; \ } @@ -314,7 +324,7 @@ rsRetVal objName##ClassExit(void) \ #define PROTOTYPEObjDebugPrint(obj) rsRetVal obj##DebugPrint(obj##_t *pThis) #define INTERFACEObjDebugPrint(obj) rsRetVal (*DebugPrint)(obj##_t *pThis) #define BEGINobjDebugPrint(obj) \ - rsRetVal obj##DebugPrint(obj##_t *pThis) \ + rsRetVal obj##DebugPrint(obj##_t __attribute__((unused)) *pThis) \ { \ DEFiRet; \ diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..45dac776 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -48,7 +48,7 @@ * * File begun on 2008-01-04 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -75,6 +75,7 @@ #include <string.h> #include <ctype.h> #include <assert.h> +#include <pthread.h> /* how many objects are supported by rsyslogd? */ #define OBJ_NUM_IDS 100 /* TODO change to a linked list? info: 16 were currently in use 2008-02-29 */ @@ -87,13 +88,18 @@ #include "modules.h" #include "errmsg.h" #include "cfsysline.h" +#include "unicode-helper.h" +#include "apc.h" +#include "datetime.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ DEFobjCurrIf(var) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +DEFobjCurrIf(strm) static objInfo_t *arrObjInfo[OBJ_NUM_IDS]; /* array with object information pointers */ +static pthread_mutex_t mutObjGlobalOp; /* mutex to guard global operations of the object system */ /* cookies for serialized lines */ @@ -144,8 +150,8 @@ InfoConstruct(objInfo_t **ppThis, uchar *pszID, int iObjVers, ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pThis->pszID = pszID; - pThis->lenID = strlen((char*)pszID); - pThis->pszName = (uchar*)strdup((char*)pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ + pThis->lenID = ustrlen(pszID); + pThis->pszName = ustrdup(pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ pThis->iObjVers = iObjVers; pThis->QueryIF = pQueryIF; pThis->pModInfo = pModInfo; @@ -176,8 +182,7 @@ InfoDestruct(objInfo_t **ppThis) pThis = *ppThis; assert(pThis != NULL); - if(pThis->pszName != NULL) - free(pThis->pszName); + free(pThis->pszName); free(pThis); *ppThis = NULL; @@ -205,9 +210,7 @@ DestructObjSelf(obj_t *pThis) DEFiRet; ISOBJ_assert(pThis); - if(pThis->pszName != NULL) { - free(pThis->pszName); - } + free(pThis->pszName); RETiRet; } @@ -228,20 +231,20 @@ static rsRetVal objSerializeHeader(strm_t *pStrm, obj_t *pObj, uchar *pszRecType assert(!strcmp((char*) pszRecType, "Obj") || !strcmp((char*) pszRecType, "OPB")); /* object cookie and serializer version (so far always 1) */ - CHKiRet(strmWriteChar(pStrm, COOKIE_OBJLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '1')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_OBJLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '1')); /* object type, version and string length */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWrite(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteLong(pStrm, objGetVersion(pObj))); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, objGetVersion(pObj))); /* record trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -259,7 +262,7 @@ BeginSerialize(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "Obj")); finalize_it: @@ -284,7 +287,7 @@ BeginSerializePropBag(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "OPB")); finalize_it: @@ -320,31 +323,31 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr switch(propType) { case PROPTYPE_PSZ: pszBuf = (uchar*) pUsr; - lenBuf = strlen((char*) pszBuf); + lenBuf = ustrlen(pszBuf); vType = VARTYPE_STR; break; case PROPTYPE_SHORT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((short*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((int*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_LONG: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((long*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT64: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((int64*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_CSTR: @@ -377,23 +380,23 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr } /* cookie */ - CHKiRet(strmWriteChar(pStrm, COOKIE_PROPLINE)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_PROPLINE)); /* name */ - CHKiRet(strmWrite(pStrm, pszPropName, strlen((char*)pszPropName))); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pszPropName, ustrlen(pszPropName))); + CHKiRet(strm.WriteChar(pStrm, ':')); /* type */ - CHKiRet(strmWriteLong(pStrm, (int) vType)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, (int) vType)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* length */ - CHKiRet(strmWriteLong(pStrm, lenBuf)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, lenBuf)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* data */ - CHKiRet(strmWrite(pStrm, (uchar*) pszBuf, lenBuf)); + CHKiRet(strm.Write(pStrm, (uchar*) pszBuf, lenBuf)); /* trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -410,12 +413,12 @@ EndSerialize(strm_t *pStrm) assert(pStrm != NULL); - CHKiRet(strmWriteChar(pStrm, COOKIE_ENDLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); - CHKiRet(strmWriteChar(pStrm, COOKIE_BLANKLINE)); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_ENDLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_BLANKLINE)); + CHKiRet(strm.WriteChar(pStrm, '\n')); - CHKiRet(strmRecordEnd(pStrm)); + CHKiRet(strm.RecordEnd(pStrm)); finalize_it: RETiRet; @@ -423,7 +426,7 @@ finalize_it: /* define a helper to make code below a bit cleaner (and quicker to write) */ -#define NEXTC CHKiRet(strmReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ +#define NEXTC CHKiRet(strm.ReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ /* de-serialize an embedded, non-octect-counted string. This is useful @@ -440,20 +443,20 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) assert(ppStr != NULL); - CHKiRet(rsCStrConstruct(&pStr)); + CHKiRet(cstrConstruct(&pStr)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pStr, c)); + CHKiRet(cstrAppendChar(pStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; finalize_it: if(iRet != RS_RET_OK && pStr != NULL) - rsCStrDestruct(&pStr); + cstrDestruct(&pStr); RETiRet; } @@ -508,14 +511,14 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) assert(ppCStr != NULL); assert(iLen >= 0); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); NEXTC; for(i = 0 ; i < iLen ; ++i) { - CHKiRet(rsCStrAppendChar(pCStr, c)); + CHKiRet(cstrAppendChar(pCStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* check terminator */ if(c != ':') ABORT_FINALIZE(RS_RET_INVALID_DELIMITER); @@ -524,7 +527,7 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) finalize_it: if(iRet != RS_RET_OK && pCStr != NULL) - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); RETiRet; } @@ -617,19 +620,19 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) NEXTC; if(c != COOKIE_PROPLINE) { /* oops, we've read one char that does not belong to use - unget it first */ - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); ABORT_FINALIZE(RS_RET_NO_PROPLINE); } /* get the property name first */ - CHKiRet(rsCStrConstruct(&pProp->pcsName)); + CHKiRet(cstrConstruct(&pProp->pcsName)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pProp->pcsName, c)); + CHKiRet(cstrAppendChar(pProp->pcsName, c)); NEXTC; } - CHKiRet(rsCStrFinish(pProp->pcsName)); + CHKiRet(cstrFinalize(pProp->pcsName)); /* property type */ CHKiRet(objDeserializeNumber(&i, pStrm)); @@ -718,7 +721,7 @@ static rsRetVal objDeserializeTryRecover(strm_t *pStrm) } } - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); finalize_it: dbgprintf("deserializer has possibly been able to re-sync and recover, state %d\n", iRet); @@ -803,7 +806,7 @@ Deserialize(void *ppObj, uchar *pszTypeExpected, strm_t *pStrm, rsRetVal (*fFixu } } while(iRetLocal != RS_RET_OK); - if(rsCStrSzStrCmp(pstrID, pszTypeExpected, strlen((char*)pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ + if(rsCStrSzStrCmp(pstrID, pszTypeExpected, ustrlen(pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ ABORT_FINALIZE(RS_RET_INVALID_OID); CHKiRet(FindObjInfo(pstrID, &pObjInfo)); @@ -948,13 +951,8 @@ SetName(obj_t *pThis, uchar *pszName) { DEFiRet; - if(pThis->pszName != NULL) - free(pThis->pszName); - - pThis->pszName = (uchar*) strdup((char*) pszName); - - if(pThis->pszName == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); finalize_it: RETiRet; @@ -1057,7 +1055,7 @@ RegisterObj(uchar *pszObjName, objInfo_t *pInfo) i = 0; while(!bFound && i < OBJ_NUM_IDS && arrObjInfo[i] != NULL) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } @@ -1096,7 +1094,7 @@ UnregisterObj(uchar *pszObjName) i = 0; while(!bFound && i < OBJ_NUM_IDS) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } @@ -1132,6 +1130,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* DEV debug only: dbgprintf("source file %s requests object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ + pthread_mutex_lock(&mutObjGlobalOp); if(pIf->ifIsLoaded == 1) { ABORT_FINALIZE(RS_RET_OK); /* we are already set */ @@ -1172,6 +1171,8 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 1; /* we are happy */ finalize_it: + pthread_mutex_unlock(&mutObjGlobalOp); + if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1193,15 +1194,16 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* dev debug only dbgprintf("source file %s releasing object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ + pthread_mutex_lock(&mutObjGlobalOp); if(pObjFile == NULL) FINALIZE; /* if it is not a lodable module, we do not need to do anything... */ if(pIf->ifIsLoaded == 0) { - ABORT_FINALIZE(RS_RET_OK); /* we are not loaded - this is perfectly OK... */ + FINALIZE; /* we are not loaded - this is perfectly OK... */ } else if(pIf->ifIsLoaded == 2) { pIf->ifIsLoaded = 0; /* clean up */ - ABORT_FINALIZE(RS_RET_OK); /* we had a load error and can not continue */ + FINALIZE; /* we had a load error and can not/must not continue */ } CHKiRet(rsCStrConstructFromszStr(&pStr, pObjName)); @@ -1213,6 +1215,8 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 0; /* indicated "no longer valid" */ finalize_it: + pthread_mutex_unlock(&mutObjGlobalOp); + if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1278,14 +1282,15 @@ objClassExit(void) { DEFiRet; /* release objects we no longer need */ + objRelease(strm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); /* TODO: implement the class exits! */ #if 0 - cfsyslineInit(pModInfo); - varClassInit(pModInfo); + cfsyslineExit(pModInfo); + varClassExit(pModInfo); #endif errmsgClassExit(); moduleClassExit(); @@ -1304,8 +1309,9 @@ objClassExit(void) rsRetVal objClassInit(modInfo_t *pModInfo) { - DEFiRet; + pthread_mutexattr_t mutAttr; int i; + DEFiRet; /* first, initialize the object system itself. This must be done * before any other object is created. @@ -1314,17 +1320,28 @@ objClassInit(modInfo_t *pModInfo) arrObjInfo[i] = NULL; } + /* the mutex must be recursive, because objects may call into other + * object identifieres recursively. + */ + pthread_mutexattr_init(&mutAttr); + pthread_mutexattr_settype(&mutAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&mutObjGlobalOp, &mutAttr); + /* request objects we use */ CHKiRet(objGetObjInterface(&obj)); /* get ourselves ;) */ /* init classes we use (limit to as few as possible!) */ CHKiRet(errmsgClassInit(pModInfo)); + CHKiRet(datetimeClassInit(pModInfo)); + CHKiRet(apcClassInit(pModInfo)); CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); CHKiRet(moduleClassInit(pModInfo)); + CHKiRet(strmClassInit(pModInfo)); CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); finalize_it: RETiRet; diff --git a/runtime/obj.h b/runtime/obj.h index dc04203b..419d29cc 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -68,7 +68,7 @@ #define objSerializePTR(strm, propName, propType) \ CHKiRet(obj.SerializeProp(strm, (uchar*) #propName, PROPTYPE_##propType, (void*) pThis->propName)); #define DEFobjStaticHelpers \ - static objInfo_t *pObjInfoOBJ = NULL; \ + static objInfo_t __attribute__((unused)) *pObjInfoOBJ = NULL; \ DEFobjCurrIf(obj) @@ -77,11 +77,13 @@ /* the next macro MUST be called in Constructors: */ #ifndef NDEBUG /* this means if debug... */ # define objConstructSetObjInfo(pThis) \ - ASSERT(((obj_t*) (pThis))->pObjInfo == NULL); \ ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ; \ + ((obj_t*) (pThis))->pszName = NULL; \ ((obj_t*) (pThis))->iObjCooCKiE = 0xBADEFEE #else -# define objConstructSetObjInfo(pThis) ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ +# define objConstructSetObjInfo(pThis) \ + ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ; \ + ((obj_t*) (pThis))->pszName = NULL #endif #define objDestruct(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_DESTRUCT])(&pThis) #define objSerialize(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_SERIALIZE]) diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 8dddc4b8..1d442c61 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -67,31 +67,25 @@ rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries) assert(ppThis != NULL); assert(iNumEntries >= 0); - if((pThis = calloc(1, sizeof(omodStringRequest_t))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis = calloc(1, sizeof(omodStringRequest_t))); /* got the structure, so fill it */ pThis->iNumEntries = iNumEntries; /* allocate string for template name array. The individual strings will be * allocated as the code progresses (we do not yet know the string sizes) */ - if((pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))); + /* allocate the template options array. */ - if((pThis->piTplOpts = calloc(iNumEntries, sizeof(int))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->piTplOpts = calloc(iNumEntries, sizeof(int))); -abort_it: +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis != NULL) { + OMSRdestruct(pThis); + pThis = NULL; + } + } *ppThis = pThis; RETiRet; } @@ -155,7 +149,7 @@ OMSRgetSupportedTplOpts(unsigned long *pOpts) { DEFiRet; assert(pOpts != NULL); - *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY; + *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY | OMSR_TPL_AS_MSG; RETiRet; } diff --git a/runtime/objomsr.h b/runtime/objomsr.h index 75ad0fb8..e59b774f 100644 --- a/runtime/objomsr.h +++ b/runtime/objomsr.h @@ -27,8 +27,12 @@ /* define flags for required template options */ #define OMSR_NO_RQD_TPL_OPTS 0 #define OMSR_RQD_TPL_OPT_SQL 1 +/* only one of OMSR_TPL_AS_ARRAY or _AS_MSG must be specified, if both are given + * results are unpredictable. + */ #define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */ -/* next option is 4, 8, 16, ... */ +#define OMSR_TPL_AS_MSG 4 /* introduced in 5.3.4, 2009-11-02 */ +/* next option is 8, 16, 32, ... */ struct omodStringRequest_s { /* strings requested by output module for doAction() */ int iNumEntries; /* number of array entries for data elements below */ diff --git a/runtime/parser.c b/runtime/parser.c index 212d40f3..ca31b35d 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -37,7 +37,13 @@ #include "dirty.h" #include "msg.h" #include "obj.h" +#include "datetime.h" #include "errmsg.h" +#include "parser.h" +#include "ruleset.h" +#include "unicode-helper.h" +#include "dirty.h" +#include "cfsysline.h" /* some defines */ #define DEFUPRI (LOG_USER|LOG_NOTICE) @@ -46,26 +52,167 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) +DEFobjCurrIf(ruleset) /* static data */ +/* config data */ +static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ +static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */ +static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ +static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ + +/* This is the list of all parsers known to us. + * This is also used to unload all modules on shutdown. + */ +parserList_t *pParsLstRoot = NULL; + +/* this is the list of the default parsers, to be used if no others + * are specified. + */ +parserList_t *pDfltParsLst = NULL; + + +/* intialize (but NOT allocate) a parser list. Primarily meant as a hook + * which can be used to extend the list in the future. So far, just sets + * it to NULL. + */ +static rsRetVal +InitParserList(parserList_t **pListRoot) +{ + *pListRoot = NULL; + return RS_RET_OK; +} + -/* this is a dummy class init +/* destruct a parser list. The list elements are destroyed, but the parser objects + * themselves are not modified. (That is done at a late stage during rsyslogd + * shutdown and need not be considered here.) */ -rsRetVal parserClassInit(void) +static rsRetVal +DestructParserList(parserList_t **ppListRoot) { + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = *ppListRoot; + while(pParsLst != NULL) { + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } + *ppListRoot = NULL; + return RS_RET_OK; +} + + +/* Add a parser to the list. We use a VERY simple and ineffcient algorithm, + * but it is employed only for a few milliseconds during config processing. So + * I prefer to keep it very simple and with simple data structures. Unfortunately, + * we need to preserve the order, but I don't like to add a tail pointer as that + * would require a container object. So I do the extra work to skip to the tail + * when adding elements... + * rgerhards, 2009-11-03 + */ +static rsRetVal +AddParserToList(parserList_t **ppListRoot, parser_t *pParser) +{ + parserList_t *pThis; + parserList_t *pTail; DEFiRet; - /* request objects we use */ - CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */ - CHKiRet(objUse(glbl, CORE_COMPONENT)); - CHKiRet(objUse(errmsg, CORE_COMPONENT)); -// TODO: free components! see action.c + CHKmalloc(pThis = MALLOC(sizeof(parserList_t))); + pThis->pParser = pParser; + pThis->pNext = NULL; + + if(*ppListRoot == NULL) { + pThis->pNext = *ppListRoot; + *ppListRoot = pThis; + } else { + /* find tail first */ + for(pTail = *ppListRoot ; pTail->pNext != NULL ; pTail = pTail->pNext) + /* just search, do nothing else */; + /* add at tail */ + pTail->pNext = pThis; + } + finalize_it: RETiRet; } +/* find a parser based on the provided name */ +static rsRetVal +FindParser(parser_t **ppParser, uchar *pName) +{ + parserList_t *pThis; + DEFiRet; + + for(pThis = pParsLstRoot ; pThis != NULL ; pThis = pThis->pNext) { + if(ustrcmp(pThis->pParser->pName, pName) == 0) { + *ppParser = pThis->pParser; + FINALIZE; /* found it, iRet still eq. OK! */ + } + } + + iRet = RS_RET_PARSER_NOT_FOUND; + +finalize_it: + RETiRet; +} + + +/* --- END helper functions for parser list handling --- */ + +/* Add a an already existing parser to the default list. As usual, order + * of calls is important (most importantly, that means the legacy parser, + * which can process everything, MUST be added last!). + * rgerhards, 2009-11-04 + */ +static rsRetVal +AddDfltParser(uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + CHKiRet(FindParser(&pParser, pName)); + CHKiRet(AddParserToList(&pDfltParsLst, pParser)); + dbgprintf("Parser '%s' added to default parser set.\n", pName); + +finalize_it: + RETiRet; +} + + + +BEGINobjConstruct(parser) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(parser) + +/* ConstructionFinalizer. The most important chore is to add the parser object + * to our global list of available parsers. + * rgerhards, 2009-11-03 + */ +rsRetVal parserConstructFinalize(parser_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + CHKiRet(AddParserToList(&pParsLstRoot, pThis)); + DBGPRINTF("Parser '%s' added to list of available parsers.\n", pThis->pName); + +finalize_it: + RETiRet; +} + +BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(parser) + dbgprintf("destructing parser '%s'\n", pThis->pName); + free(pThis->pName); +ENDobjDestruct(parser) + + /* uncompress a received message if it is compressed. * pMsg->pszRawMsg buffer is updated. * rgerhards, 2008-10-09 @@ -96,7 +243,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) */ int ret; iLenDefBuf = glbl.GetMaxLine(); - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1))); + CHKmalloc(deflateBuf = MALLOC(sizeof(uchar) * (iLenDefBuf + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, (int) (lenMsg-1)); @@ -114,10 +261,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) "Message ignored.", ret); FINALIZE; /* unconditional exit, nothing left to do... */ } - free(pMsg->pszRawMsg); - pMsg->pszRawMsg = deflateBuf; - pMsg->iLenRawMsg = iLenDefBuf; - deflateBuf = NULL; /* logically "freed" - caller is now responsible */ + MsgSetRawMsg(pMsg, (char*)deflateBuf, iLenDefBuf); } finalize_it: if(deflateBuf != NULL) @@ -156,7 +300,7 @@ finalize_it: * rgerhards, 2007-09-14 */ static inline rsRetVal -sanitizeMessage(msg_t *pMsg) +SanitizeMsg(msg_t *pMsg) { DEFiRet; uchar *pszMsg; @@ -165,12 +309,12 @@ sanitizeMessage(msg_t *pMsg) size_t iSrc; size_t iDst; size_t iMaxLine; + size_t maxDest; + sbool bUpdatedLen = FALSE; + uchar szSanBuf[32*1024]; /* buffer used for sanitizing a string */ assert(pMsg != NULL); - -# ifdef USE_NETZIP - CHKiRet(uncompressMessage(pMsg)); -# endif + assert(pMsg->iLenRawMsg > 0); pszMsg = pMsg->pszRawMsg; lenMsg = pMsg->iLenRawMsg; @@ -178,6 +322,7 @@ sanitizeMessage(msg_t *pMsg) /* remove NUL character at end of message (see comment in function header) */ if(pszMsg[lenMsg-1] == '\0') { DBGPRINTF("dropped NUL at very end of message\n"); + bUpdatedLen = TRUE; lenMsg--; } @@ -188,89 +333,112 @@ sanitizeMessage(msg_t *pMsg) */ if(bDropTrailingLF && pszMsg[lenMsg-1] == '\n') { DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); + bUpdatedLen = TRUE; lenMsg--; } + /* it is much quicker to sweep over the message and see if it actually + * needs sanitation than to do the sanitation in any case. So we first do + * this and terminate when it is not needed - which is expectedly the case + * for the vast majority of messages. -- rgerhards, 2009-06-15 + * Note that we do NOT check here if tab characters are to be escaped or + * not. I expect this functionality to be seldomly used and thus I do not + * like to pay the performance penalty. So the penalty is only with those + * that actually use it, because we may call the sanitizer without actual + * need below (but it then still will work perfectly well!). -- rgerhards, 2009-11-27 + */ + int bNeedSanitize = 0; + for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { + if(iscntrl(pszMsg[iSrc])) { + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + bNeedSanitize = 1; + break; + } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + bNeedSanitize = 1; + break; + } + } + + if(!bNeedSanitize) { + if(bUpdatedLen == TRUE) + MsgSetRawMsgSize(pMsg, lenMsg); + FINALIZE; + } + /* now copy over the message and sanitize it */ - /* TODO: can we get cheaper memory alloc? {alloca()?}*/ iMaxLine = glbl.GetMaxLine(); - CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + maxDest = lenMsg * 4; /* message can grow at most four-fold */ + if(maxDest > iMaxLine) + maxDest = iMaxLine; /* but not more than the max size! */ + if(maxDest < sizeof(szSanBuf)) + pDst = szSanBuf; + else + CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; - while(iSrc < lenMsg && iDst < iMaxLine) { - if(pszMsg[iSrc] == '\0') { /* guard against \0 characters... */ - /* changed to the sequence (somewhat) proposed in - * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 - */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ - pDst[iDst++] = cCCEscapeChar; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - } /* if we do not have space, we simply ignore the '\0'... */ - /* log an error? Very questionable... rgerhards, 2006-11-30 */ - /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ - } else if(bEscapeCCOnRcv && iscntrl((int) pszMsg[iSrc])) { - /* we are configured to escape control characters. Please note - * that this most probably break non-western character sets like - * Japanese, Korean or Chinese. rgerhards, 2007-07-17 - * Note: sysklogd logs octal values only for DEL and CCs above 127. - * For others, it logs ^n where n is the control char converted to an - * alphabet character. We like consistency and thus escape it to octal - * in all cases. If someone complains, we may change the mode. At least - * we known now what's going on. - * rgerhards, 2007-07-17 + while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ + if(iscntrl((int) pszMsg[iSrc]) && (pszMsg[iSrc] != '\t' || bEscapeTab)) { + /* note: \0 must always be escaped, the rest of the code currently + * can not handle it! -- rgerhards, 2009-08-26 */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + /* we are configured to escape control characters. Please note + * that this most probably break non-western character sets like + * Japanese, Korean or Chinese. rgerhards, 2007-07-17 + */ pDst[iDst++] = cCCEscapeChar; pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); - } /* again, if we do not have space, we ignore the char - see comment at '\0' */ + } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + /* In this case, we also do the conversion. Note that this most + * probably breaks European languages. -- rgerhards, 2010-01-27 + */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); } else { pDst[iDst++] = pszMsg[iSrc]; } ++iSrc; } - pDst[iDst] = '\0'; /* space *is* reserved for this! */ - /* we have a sanitized string. Let's save it now */ - free(pMsg->pszRawMsg); - if((pMsg->pszRawMsg = malloc((iDst+1) * sizeof(uchar))) == NULL) { - /* when we get no new buffer, we use what we already have ;) */ - pMsg->pszRawMsg = pDst; - } else { - /* trim buffer */ - memcpy(pMsg->pszRawMsg, pDst, iDst+1); - free(pDst); /* too big! */ - pMsg->iLenRawMsg = iDst; - } + MsgSetRawMsg(pMsg, (char*)pDst, iDst); /* save sanitized string */ + + if(pDst != szSanBuf) + free(pDst); finalize_it: RETiRet; } -/* Parse a received message. The object's rawmsg property is taken and - * parsed according to the relevant standards. This can later be - * extended to support configured parsers. - * rgerhards, 2008-10-09 +/* A standard parser to parse out the PRI. This is made available in + * this module as it is expected that allmost all parsers will need + * that functionality and so they do not need to implement it themsleves. */ -rsRetVal parseMsg(msg_t *pMsg) +static inline rsRetVal +ParsePRI(msg_t *pMsg) { - DEFiRet; - uchar *msg; int pri; - - CHKiRet(sanitizeMessage(pMsg)); - - /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ - DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); + uchar *msg; + int lenMsg; + int iPriText; + DEFiRet; /* pull PRI */ - pri = DEFUPRI; + lenMsg = pMsg->iLenRawMsg; msg = pMsg->pszRawMsg; + pri = DEFUPRI; + iPriText = 0; if(*msg == '<') { + /* while we process the PRI, we also fill the PRI textual representation + * inside the msg object. This may not be ideal from an OOP point of view, + * but it offers us performance... + */ pri = 0; - while(isdigit((int) *++msg)) { + while(--lenMsg > 0 && isdigit((int) *++msg)) { pri = 10 * pri + (*msg - '0'); } if(*msg == '>') @@ -280,36 +448,253 @@ rsRetVal parseMsg(msg_t *pMsg) } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - MsgSetUxTradMsg(pMsg, (char*) msg); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); + RETiRet; +} - if(pMsg->bParseHOSTNAME == 0) - MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); - /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have - * a traditional syslog message or one formatted according to syslog-protocol. - * We need to apply different parsers depending on that. We use the - * -protocol VERSION field for the detection. +/* Parse a received message. The object's rawmsg property is taken and + * parsed according to the relevant standards. This can later be + * extended to support configured parsers. + * rgerhards, 2008-10-09 + */ +static rsRetVal +ParseMsg(msg_t *pMsg) +{ + rsRetVal localRet; + parserList_t *pParserList; + parser_t *pParser; + sbool bIsSanitized; + sbool bPRIisParsed; + static int iErrMsgRateLimiter = 0; + DEFiRet; + + if(pMsg->iLenRawMsg == 0) + ABORT_FINALIZE(RS_RET_EMPTY_MSG); + +# ifdef USE_NETZIP + CHKiRet(uncompressMessage(pMsg)); +# endif + + /* we take the risk to print a non-sanitized string, because this is the best we can get + * (and that functionality is too important for debugging to drop it...). + */ + DBGPRINTF("msg parser: flags %x, from '%s', msg '%.50s'\n", pMsg->msgFlags, + (pMsg->msgFlags & NEEDS_DNSRESOL) ? UCHAR_CONSTANT("~NOTRESOLVED~") : getRcvFrom(pMsg), + pMsg->pszRawMsg); + + /* we now need to go through our list of parsers and see which one is capable of + * parsing the message. Note that the first parser that requires message sanitization + * will cause it to happen. After that, access to the unsanitized message is no + * loger possible. */ - if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); - setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + pParserList = ruleset.GetParserList(pMsg); + if(pParserList == NULL) { + pParserList = pDfltParsLst; + } + DBGPRINTF("parse using parser list %p%s.\n", pParserList, + (pParserList == pDfltParsLst) ? " (the default list)" : ""); + + bIsSanitized = FALSE; + bPRIisParsed = FALSE; + while(pParserList != NULL) { + pParser = pParserList->pParser; + if(pParser->bDoSanitazion && bIsSanitized == FALSE) { + CHKiRet(SanitizeMsg(pMsg)); + if(pParser->bDoPRIParsing && bPRIisParsed == FALSE) { + CHKiRet(ParsePRI(pMsg)); + bPRIisParsed = TRUE; + } + bIsSanitized = TRUE; } - } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); - setProtocolVersion(pMsg, 0); - if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + localRet = pParser->pModule->mod.pm.parse(pMsg); + dbgprintf("Parser '%s' returned %d\n", pParser->pName, localRet); + if(localRet != RS_RET_COULD_NOT_PARSE) + break; + pParserList = pParserList->pNext; + } + + /* We need to log a warning message and drop the message if we did not find a parser. + * Note that we log at most the first 1000 message, as this may very well be a problem + * that causes a message generation loop. We do not synchronize that counter, it doesn't + * matter if we log a handful messages more than we should... + */ + if(localRet != RS_RET_OK) { + if(++iErrMsgRateLimiter > 1000) { + errmsg.LogError(0, localRet, "Error: one message could not be processed by " + "any parser, message is being discarded (start of raw msg: '%.50s')", + pMsg->pszRawMsg); } + DBGPRINTF("No parser could process the message (state %d), we need to discard it.\n", localRet); + ABORT_FINALIZE(localRet); } - /* finalize message object */ + /* "finalize" message object */ pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */ - MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */ finalize_it: RETiRet; } + +/* set the parser name - string is copied over, call can continue to use it, + * but must free it if desired. + */ +static rsRetVal +SetName(parser_t *pThis, uchar *name) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + assert(name != NULL); + + if(pThis->pName != NULL) { + free(pThis->pName); + pThis->pName = NULL; + } + + CHKmalloc(pThis->pName = ustrdup(name)); + +finalize_it: + RETiRet; +} + + +/* set a pointer to "our" module. Note that no module + * pointer must already be set. + */ +static rsRetVal +SetModPtr(parser_t *pThis, modInfo_t *pMod) +{ + ISOBJ_TYPE_assert(pThis, parser); + assert(pMod != NULL); + assert(pThis->pModule == NULL); + pThis->pModule = pMod; + return RS_RET_OK; +} + + +/* Specify if we should do standard message sanitazion before we pass the data + * down to the parser. + */ +static rsRetVal +SetDoSanitazion(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoSanitazion = bDoIt; + return RS_RET_OK; +} + + +/* Specify if we should do standard PRI parsing before we pass the data + * down to the parser module. + */ +static rsRetVal +SetDoPRIParsing(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoPRIParsing = bDoIt; + return RS_RET_OK; +} + + +/* queryInterface function-- rgerhards, 2009-11-03 + */ +BEGINobjQueryInterface(parser) +CODESTARTobjQueryInterface(parser) + if(pIf->ifVersion != parserCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = parserConstruct; + pIf->ConstructFinalize = parserConstructFinalize; + pIf->Destruct = parserDestruct; + pIf->SetName = SetName; + pIf->SetModPtr = SetModPtr; + pIf->SetDoSanitazion = SetDoSanitazion; + pIf->SetDoPRIParsing = SetDoPRIParsing; + pIf->ParseMsg = ParseMsg; + pIf->SanitizeMsg = SanitizeMsg; + pIf->InitParserList = InitParserList; + pIf->DestructParserList = DestructParserList; + pIf->AddParserToList = AddParserToList; + pIf->AddDfltParser = AddDfltParser; + pIf->FindParser = FindParser; +finalize_it: +ENDobjQueryInterface(parser) + + + +/* Reset config variables to default values. + * rgerhards, 2007-07-17 + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + cCCEscapeChar = '#'; + bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bEscape8BitChars = 0; /* default is to escape control characters */ + bEscapeTab = 1; /* default is to escape control characters */ + bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ + + return RS_RET_OK; +} + +/* This destroys the master parserlist and all of its parser entries. MUST only be + * done when the module is shut down. Parser modules are NOT unloaded, rsyslog + * does that at a later stage for all dynamically loaded modules. + */ +static void +destroyMasterParserList(void) +{ + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = pParsLstRoot; + while(pParsLst != NULL) { + parserDestruct(&pParsLst->pParser); + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } +} + +/* Exit our class. + * rgerhards, 2009-11-04 + */ +BEGINObjClassExit(parser, OBJ_IS_CORE_MODULE) /* class, version */ + DestructParserList(&pDfltParsLst); + destroyMasterParserList(); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDObjClassExit(parser) + + +/* Initialize the parser class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2009-11-02 + */ +BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + + CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); + + InitParserList(&pParsLstRoot); + InitParserList(&pDfltParsLst); +ENDObjClassInit(parser) + diff --git a/runtime/parser.h b/runtime/parser.h index cec9c083..bdd572cb 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -1,8 +1,6 @@ /* header for parser.c - * This is not yet an object, but contains all those code necessary to - * parse syslog messages. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -21,10 +19,53 @@ * * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. */ -#ifndef INCLUDED_PARSE_H -#define INCLUDED_PARSE_H +#ifndef INCLUDED_PARSER_H +#define INCLUDED_PARSER_H -extern rsRetVal parserClassInit(void); -extern rsRetVal parseMsg(msg_t*); -#endif /* #ifndef INCLUDED_PARSE_H */ +/* we create a small helper object, a list of parsers, that we can use to + * build a chain of them whereever this is needed (initially thought to be + * used in ruleset.c as well as ourselvs). + */ +struct parserList_s { + parser_t *pParser; + parserList_t *pNext; +}; + + +/* the parser object, a dummy because we have only static methods */ +struct parser_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *pName; /* name of this parser */ + modInfo_t *pModule; /* pointer to parser's module */ + sbool bDoSanitazion; /* do standard message sanitazion before calling parser? */ + sbool bDoPRIParsing; /* do standard PRI parsing before calling parser? */ +}; + +/* interfaces */ +BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(var); + rsRetVal (*Construct)(parser_t **ppThis); + rsRetVal (*ConstructFinalize)(parser_t *pThis); + rsRetVal (*Destruct)(parser_t **ppThis); + rsRetVal (*SetName)(parser_t *pThis, uchar *name); + rsRetVal (*SetModPtr)(parser_t *pThis, modInfo_t *pMod); + rsRetVal (*SetDoSanitazion)(parser_t *pThis, int); + rsRetVal (*SetDoPRIParsing)(parser_t *pThis, int); + rsRetVal (*FindParser)(parser_t **ppThis, uchar*name); + rsRetVal (*InitParserList)(parserList_t **pListRoot); + rsRetVal (*DestructParserList)(parserList_t **pListRoot); + rsRetVal (*AddParserToList)(parserList_t **pListRoot, parser_t *pParser); + /* static functions */ + rsRetVal (*ParseMsg)(msg_t *pMsg); + rsRetVal (*SanitizeMsg)(msg_t *pMsg); + rsRetVal (*AddDfltParser)(uchar *); +ENDinterface(parser) +#define parserCURR_IF_VERSION 1 /* increment whenever you change the interface above! */ + + +/* prototypes */ +PROTOTYPEObj(parser); + + +#endif /* #ifndef INCLUDED_PARSER_H */ diff --git a/runtime/prop.c b/runtime/prop.c new file mode 100644 index 00000000..94d1bd49 --- /dev/null +++ b/runtime/prop.c @@ -0,0 +1,247 @@ +/* prop.c - rsyslog's prop object + * + * This object is meant to support message properties that are stored + * seperately from the message. The main intent is to support properties + * that are "constant" during a period of time, so that many messages may + * contain a reference to the same property. It is important, though, that + * properties are destroyed when they are no longer needed. + * + * Please note that this is a performance-critical part of the software and + * as such we may use some methods in here which do not look elegant, but + * which are fast... + * + * Module begun 2009-06-17 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <assert.h> +#include <string.h> + +#include "rsyslog.h" +#include "obj.h" +#include "obj-types.h" +#include "unicode-helper.h" +#include "atomic.h" +#include "prop.h" + +/* static data */ +DEFobjStaticHelpers + + +/* Standard-Constructor + */ +BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! */ + pThis->iRefCount = 1; +ENDobjConstruct(prop) + + +/* destructor for the prop object */ +BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ + int currRefCount; +CODESTARTobjDestruct(prop) + currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); + if(currRefCount == 0) { + /* (only) in this case we need to actually destruct the object */ + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + } else { + pThis = NULL; /* tell framework NOT to destructing the object! */ + } +ENDobjDestruct(prop) + +/* set string, we make our own private copy! This MUST only be called BEFORE + * ConstructFinalize()! + */ +static rsRetVal SetString(prop_t *pThis, uchar *psz, int len) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + pThis->len = len; + if(len < CONF_PROP_BUFSIZE) { + memcpy(pThis->szVal.sz, psz, len + 1); + } else { + CHKmalloc(pThis->szVal.psz = MALLOC(len + 1)); + memcpy(pThis->szVal.psz, psz, len + 1); + } + +finalize_it: + RETiRet; +} + + +/* get string length */ +static int GetStringLen(prop_t *pThis) +{ + return pThis->len; +} + + +/* get string */ +static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) +{ + BEGINfunc + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len < CONF_PROP_BUFSIZE) { + *ppsz = pThis->szVal.sz; + } else { + *ppsz = pThis->szVal.psz; + } + *plen = pThis->len; + ENDfunc + return RS_RET_OK; +} + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +propConstructFinalize(prop_t __attribute__((unused)) *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + RETiRet; +} + + +/* add a new reference. It is VERY IMPORTANT to call this function whenever + * the property is handed over to some entitiy that later call Destruct() on it. + */ +static rsRetVal AddRef(prop_t *pThis) +{ + ATOMIC_INC(pThis->iRefCount); + return RS_RET_OK; +} + + +/* this is a "do it all in one shot" function that creates a new property, + * assigns the provided string to it and finalizes the property. Among the + * convenience, it is alos (very, very) slightly faster. + * rgerhards, 2009-07-01 + */ +static rsRetVal CreateStringProp(prop_t **ppThis, uchar* psz, int len) +{ + DEFiRet; + propConstruct(ppThis); + SetString(*ppThis, psz, len); + propConstructFinalize(*ppThis); + RETiRet; +} + +/* another one-stop function, quite useful: it takes a property pointer and + * a string. If the string is already contained in the property, nothing happens. + * If the string is different (or the pointer NULL), the current property + * is destructed and a new one created. This can be used to get a specific + * name in those cases where there is a good chance that the property + * immediatly previously processed already contained the value we need - in + * which case we save us all the creation overhead by just reusing the already + * existing property). + * rgerhards, 2009-07-01 + */ +rsRetVal CreateOrReuseStringProp(prop_t **ppThis, uchar *psz, int len) +{ + uchar *pszPrev; + int lenPrev; + DEFiRet; + assert(ppThis != NULL); + + if(*ppThis == NULL) { + /* we need to create a property */ + CHKiRet(CreateStringProp(ppThis, psz, len)); + } else { + /* already exists, check if we can re-use it */ + GetString(*ppThis, &pszPrev, &lenPrev); + if(len != lenPrev || ustrcmp(psz, pszPrev)) { + /* different, need to discard old & create new one */ + propDestruct(ppThis); + CHKiRet(CreateStringProp(ppThis, psz, len)); + } /* else we can re-use the existing one! */ + } + +finalize_it: + RETiRet; +} + + +/* debugprint for the prop object */ +BEGINobjDebugPrint(prop) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(prop) + dbgprintf("prop object %p - no further debug info implemented\n", pThis); +ENDobjDebugPrint(prop) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(prop) +CODESTARTobjQueryInterface(prop) + if(pIf->ifVersion != propCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = propConstruct; + pIf->ConstructFinalize = propConstructFinalize; + pIf->Destruct = propDestruct; + pIf->DebugPrint = propDebugPrint; + pIf->SetString = SetString; + pIf->GetString = GetString; + pIf->GetStringLen = GetStringLen; + pIf->AddRef = AddRef; + pIf->CreateStringProp = CreateStringProp; + pIf->CreateOrReuseStringProp = CreateOrReuseStringProp; + +finalize_it: +ENDobjQueryInterface(prop) + + +/* Exit the prop class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(prop, OBJ_IS_CORE_MODULE) /* class, version */ +// objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(prop) + + +/* Initialize the prop class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(prop, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ +// CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, propDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, propConstructFinalize); +ENDObjClassInit(prop) + +/* vi:set ai: + */ diff --git a/runtime/prop.h b/runtime/prop.h new file mode 100644 index 00000000..e3519664 --- /dev/null +++ b/runtime/prop.h @@ -0,0 +1,58 @@ +/* The prop object. + * + * This implements props within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_PROP_H +#define INCLUDED_PROP_H + +/* the prop object */ +struct prop_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int iRefCount; /* reference counter */ + union { + uchar *psz; /* stored string */ + uchar sz[CONF_PROP_BUFSIZE]; + } szVal; + int len; /* we use int intentionally, otherwise we may get some troubles... */ +}; + +/* interfaces */ +BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(prop); + rsRetVal (*Construct)(prop_t **ppThis); + rsRetVal (*ConstructFinalize)(prop_t *pThis); + rsRetVal (*Destruct)(prop_t **ppThis); + rsRetVal (*SetString)(prop_t *pThis, uchar* psz, int len); + rsRetVal (*GetString)(prop_t *pThis, uchar** ppsz, int *plen); + int (*GetStringLen)(prop_t *pThis); + rsRetVal (*AddRef)(prop_t *pThis); + rsRetVal (*CreateStringProp)(prop_t **ppThis, uchar* psz, int len); + rsRetVal (*CreateOrReuseStringProp)(prop_t **ppThis, uchar *psz, int len); +ENDinterface(prop) +#define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(prop); + +#endif /* #ifndef INCLUDED_PROP_H */ diff --git a/runtime/queue.c b/runtime/queue.c index 4e017e84..b29ec7ac 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -8,7 +8,11 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * NOTE: as of 2009-04-22, I have begin to remove the qqueue* prefix from static + * function names - this makes it really hard to read and does not provide much + * benefit, at least I (now) think so... + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -49,71 +53,175 @@ #include "obj.h" #include "wtp.h" #include "wti.h" +#include "msg.h" #include "atomic.h" +#include "errmsg.h" +#include "datetime.h" +#include "unicode-helper.h" +#include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS # include <sched.h> -# define pthread_yield() sched_yield() #endif /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) +DEFobjCurrIf(strm) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) /* forward-definitions */ -rsRetVal qqueueChkPersist(qqueue_t *pThis); -static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); -static rsRetVal qqueueRateLimiter(qqueue_t *pThis); +static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates); +static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); -static int qqueueIsIdleDA(qqueue_t *pThis); -static rsRetVal qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); -static rsRetVal qqueueConsumerCancelCleanup(void *arg1, void *arg2); -static rsRetVal qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); +static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); +static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti); +static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 #define QUEUE_NO_CHECKPOINT 0 +/*********************************************************************** + * we need a private data structure, the "to-delete" list. As C does + * not provide any partly private data structures, we implement this + * structure right here inside the module. + * Note that this list must always be kept sorted based on a unique + * dequeue ID (which is monotonically increasing). + * rgerhards, 2009-05-18 + ***********************************************************************/ + +/* generate next uniqueue dequeue ID. Note that uniqueness is only required + * on a per-queue basis and while this instance runs. So a stricly monotonically + * increasing counter is sufficient (if enough bits are used). + */ +static inline qDeqID getNextDeqID(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->deqIDAdd++; +} + + +/* return the top element of the to-delete list or NULL, if the + * list is empty. + */ +static inline toDeleteLst_t *tdlPeek(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->toDeleteLst; +} + + +/* remove the top element of the to-delete list. Nothing but the + * element itself is destroyed. Must not be called when the list + * is empty. + */ +static inline rsRetVal tdlPop(qqueue_t *pQueue) +{ + toDeleteLst_t *pRemove; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + pRemove = pQueue->toDeleteLst; + pQueue->toDeleteLst = pQueue->toDeleteLst->pNext; + free(pRemove); + + RETiRet; +} + + +/* Add a new to-delete list entry. The function allocates the data + * structure, populates it with the values provided and links the new + * element into the correct place inside the list. + */ +static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElemDeq) +{ + toDeleteLst_t *pNew; + toDeleteLst_t *pPrev; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + CHKmalloc(pNew = MALLOC(sizeof(toDeleteLst_t))); + pNew->deqID = deqID; + pNew->nElemDeq = nElemDeq; + + /* now find right spot */ + for( pPrev = pQueue->toDeleteLst + ; pPrev != NULL && deqID > pPrev->deqID + ; pPrev = pPrev->pNext) { + /*JUST SEARCH*/; + } + + if(pPrev == NULL) { + pNew->pNext = pQueue->toDeleteLst; + pQueue->toDeleteLst = pNew; + } else { + pNew->pNext = pPrev->pNext; + pPrev->pNext = pNew; + } + +finalize_it: + RETiRet; +} + + /* methods */ -/* get the overall queue size, which includes ungotten objects. Must only be called +/* get the physical queue size. Must only be called * while mutex is locked! * rgerhards, 2008-01-29 */ static inline int -qqueueGetOverallQueueSize(qqueue_t *pThis) +getPhysicalQueueSize(qqueue_t *pThis) { -#if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */ -BEGINfunc -dbgoprint((obj_t*) pThis, "queue size: %d (regular %d, ungotten %d)\n", - pThis->iQueueSize + pThis->iUngottenObjs, pThis->iQueueSize, pThis->iUngottenObjs); -ENDfunc -#endif - return pThis->iQueueSize + pThis->iUngottenObjs; + return pThis->iQueueSize; +} + + +/* get the logical queue size (that is store size minus logically dequeued elements). + * Must only be called while mutex is locked! + * rgerhards, 2009-05-19 + */ +static inline int +getLogicalQueueSize(qqueue_t *pThis) +{ + return pThis->iQueueSize - pThis->nLogDeq; } + /* This function drains the queue in cases where this needs to be done. The most probable * reason is a HUP which needs to discard data (because the queue is configured to be lossy). * During a shutdown, this is typically not needed, as the OS frees up ressources and does * this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21 * This function returns void, as it makes no sense to communicate an error back, even if * it happens. + * This functions works "around" the regular deque mechanism, because it is only used to + * clean up (in cases where message loss is acceptable). */ static inline void queueDrain(qqueue_t *pThis) { void *pUsr; - ASSERT(pThis != NULL); + BEGINfunc + DBGOPRINT((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ - while(pThis->iQueueSize-- > 0) { - pThis->qDel(pThis, &pUsr); + while(ATOMIC_DEC_AND_FETCH(pThis->iQueueSize) > 0) { + pThis->qDeq(pThis, &pUsr); if(pUsr != NULL) { objDestruct(pUsr); } + pThis->qDel(pThis); } + ENDfunc } @@ -124,7 +232,8 @@ static inline void queueDrain(qqueue_t *pThis) * this point in time. The mutex must be locked when * ths function is called. -- rgerhards, 2008-01-25 */ -static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) +static inline rsRetVal +qqueueAdviseMaxWorkers(qqueue_t *pThis) { DEFiRet; int iMaxWorkers; @@ -132,96 +241,17 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); if(!pThis->bEnqOnly) { - if(pThis->bRunsDA) { - /* if we have not yet reached the high water mark, there is no need to start a - * worker. -- rgerhards, 2008-01-26 - */ - if(qqueueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ - } + if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) { + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } else { - if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { + if(getLogicalQueueSize(pThis) == 0) { + iMaxWorkers = 0; + } else if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { iMaxWorkers = 1; } else { - iMaxWorkers = qqueueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } - wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ - } - } - - RETiRet; -} - - -/* wait until we have a fully initialized DA queue. Sometimes, we need to - * sync with it, as we expect it for some function. - * rgerhards, 2008-02-27 - */ -static rsRetVal -qqueueWaitDAModeInitialized(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - - while(pThis->bRunsDA != 2) { - d_pthread_cond_wait(&pThis->condDAReady, pThis->mut); - } - - RETiRet; -} - - -/* Destruct DA queue. This is the last part of DA-to-normal-mode - * transistion. This is called asynchronously and some time quite a - * while after the actual transistion. The key point is that we need to - * do it at some later time, because we need to destruct the DA queue. That, - * however, can not be done in a thread that has been signalled - * This is to be called when we revert back to our own queue. - * This function must be called with the queue mutex locked (the wti - * class ensures this). - * rgerhards, 2008-01-15 - */ -static rsRetVal -qqueueTurnOffDAMode(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - - /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need - * to wait for its startup... -- rgerhards, 2008-01-25 - */ - qqueueWaitDAModeInitialized(pThis); - - /* if we need to pull any data that we still need from the (child) disk queue, - * now would be the time to do so. At present, we do not need this, but I'd like to - * keep that comment if future need arises. - */ - - /* we need to check if the DA queue is empty because the DA worker may simply have - * terminated do to no new messages arriving. That does not, however, mean that the - * DA queue is empty. If there is still data in that queue, we do nothing and leave - * that for a later incarnation of this function (it will be called multiple times - * during the lifetime of DA-mode, depending on how often the DA worker receives an - * inactivity timeout. -- rgerhards, 2008-01-25 - */ - if(pThis->pqDA->iQueueSize == 0) { - pThis->bRunsDA = 0; /* tell the world we are back in non-DA mode */ - /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, - * this will be quick. - */ - qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ - dbgoprint((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n", - iRet); - /* now we need to check if the regular queue has some messages. This may be the case - * when it is waiting that the high water mark is reached again. If so, we need to start up - * a regular worker. -- rgerhards, 2008-01-26 - */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - qqueueAdviseMaxWorkers(pThis); + wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); } } @@ -244,37 +274,26 @@ qqueueChkIsDA(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pszFilePrefix != NULL) { pThis->bIsDA = 1; - dbgoprint((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); + DBGOPRINT((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); } else { - dbgoprint((obj_t*) pThis, "is NOT disk-assisted\n"); + DBGOPRINT((obj_t*) pThis, "is NOT disk-assisted\n"); } RETiRet; } -/* Start disk-assisted queue mode. All internal settings are changed. This is supposed - * to be called from the DA worker, which must have been started before. The most important - * chore of this function is to create the DA queue object. If that function fails, - * the DA worker should return with an appropriate state, which in turn should lead to - * a re-set to non-DA mode in the Enq process. The queue mutex must be locked when this - * function is called, else a number of races will happen. - * Please note that this function may be called *while* we in DA mode. This is due to the - * fact that the DA worker calls it and the DA worker may be suspended (and restarted) due - * to inactivity timeouts. +/* Start disk-assisted queue mode. * rgerhards, 2008-01-15 */ static rsRetVal -qqueueStartDA(qqueue_t *pThis) +StartDA(qqueue_t *pThis) { DEFiRet; uchar pszDAQName[128]; ISOBJ_TYPE_assert(pThis, qqueue); - if(pThis->bRunsDA == 2) /* check if already in (fully initialized) DA mode... */ - FINALIZE; /* ... then we are already done! */ - /* create message queue */ CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); @@ -293,41 +312,25 @@ qqueueStartDA(qqueue_t *pThis) CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); + CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); - CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, pThis->toQShutdown)); CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0)); - if(pThis->toQShutdown == 0) { - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ - } else { - /* we use the shortest possible shutdown (0 is endless!) because when we run on disk AND - * have an obviously large backlog, we can't finish it in any case. So there is no point - * in holding shutdown longer than necessary. -- rgerhards, 2008-01-15 - */ - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 1)); - } iRet = qqueueStart(pThis->pqDA); /* file not found is expected, that means it is no previous QIF available */ - if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) + if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) { + errno = 0; /* else an errno is shown in errmsg! */ + errmsg.LogError(errno, iRet, "error starting up disk queue, using pure in-memory mode"); + pThis->bIsDA = 0; /* disable memory mode */ FINALIZE; /* something is wrong */ + } - /* as we are right now starting DA mode because we are so busy, it is - * extremely unlikely that any regular worker is sleeping on empty queue. HOWEVER, - * we want to be on the safe side, and so we awake anyone that is waiting - * on one. So even if the scheduler plays badly with us, things should be - * quite well. -- rgerhards, 2008-01-15 - */ - wtpWakeupWrkr(pThis->pWtpReg); /* awake all workers, but not ourselves ;) */ - - pThis->bRunsDA = 2; /* we are now in DA mode, but not fully initialized */ - pThis->bChildIsDone = 0;/* set to 1 when child's worker detect queue is finished */ - pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */ - - dbgoprint((obj_t*) pThis, "is now running in disk assisted mode, disk queue 0x%lx\n", + DBGOPRINT((obj_t*) pThis, "DA queue initialized, disk queue 0x%lx\n", qqueueGetID(pThis->pqDA)); finalize_it: @@ -335,7 +338,7 @@ finalize_it: if(pThis->pqDA != NULL) { qqueueDestruct(&pThis->pqDA); } - dbgoprint((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); + DBGOPRINT((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); pThis->bIsDA = 0; } @@ -349,8 +352,8 @@ finalize_it: * If this function fails (should not happen), DA mode is not turned on. * rgerhards, 2008-01-16 */ -static inline rsRetVal -qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +static rsRetVal +InitDA(qqueue_t *pThis, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -363,82 +366,30 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) * is intentional. We assume that when we need it once, we may also need it on another * occasion. Ressources used are quite minimal when no worker is running. * rgerhards, 2008-01-24 + * NOTE: this is the DA worker *pool*, not the DA queue! */ - if(pThis->pWtpDA == NULL) { - lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DA", obj.GetName((obj_t*) pThis)); - CHKiRet(wtpConstruct (&pThis->pWtpDA)); - CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerDA)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) qqueueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); - CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); - CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); - CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); - CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); - CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); - CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); - } + lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis)); + CHKiRet(wtpConstruct (&pThis->pWtpDA)); + CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerDA)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); + CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); + CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); + CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); + CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); + CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); + CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); /* if we reach this point, we have a "good" DA worker pool */ - /* indicate we now run in DA mode - this is reset by the DA worker if it fails */ - pThis->bRunsDA = 1; - pThis->bDAEnqOnly = bEnqOnly; - - /* now we must now adivse the wtp that we need one worker. If none is yet active, - * that will also start one up. If we forgot that step, everything would be stalled - * until the next enqueue request. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues alsways have just one worker max */ - -finalize_it: - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - RETiRet; -} - - -/* check if we need to start disk assisted mode and send some signals to - * keep it running if we are already in it. It also checks if DA mode is - * partially initialized, in which case it waits for initialization to - * complete. - * rgerhards, 2008-01-14 - */ -static inline rsRetVal -qqueueChkStrtDA(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - /* if we do not hit the high water mark, we have nothing to do */ - if(qqueueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) - ABORT_FINALIZE(RS_RET_OK); - - if(pThis->bRunsDA) { - /* then we need to signal that we are at the high water mark again. If that happens - * on our way down the queue, that doesn't matter, because then nobody is waiting - * on the condition variable. - * (Remember that a DA queue stops draining the queue once it has reached the low - * water mark and restarts it when the high water mark is reached again - this is - * what this code here is responsible for. Please note that all workers may have been - * terminated due to the inactivity timeout, thus we need to advise the pool that - * we need at least one). - */ - dbgoprint((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n", - qqueueGetOverallQueueSize(pThis)); - qqueueAdviseMaxWorkers(pThis); - } else { - /* this is the case when we are currently not running in DA mode. So it is time - * to turn it back on. - */ - dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", - qqueueGetOverallQueueSize(pThis)); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ + /* now construct the actual queue (if it does not already exist) */ + if(pThis->pqDA == NULL) { + CHKiRet(StartDA(pThis)); } finalize_it: + END_MTX_PROTECTED_OPERATIONS(pThis->mut); RETiRet; } @@ -462,10 +413,11 @@ static rsRetVal qConstructFixedArray(qqueue_t *pThis) if(pThis->iMaxQueueSize == 0) ABORT_FINALIZE(RS_RET_QSIZE_ZERO); - if((pThis->tVars.farray.pBuf = malloc(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { + if((pThis->tVars.farray.pBuf = MALLOC(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } + pThis->tVars.farray.deqhead = 0; pThis->tVars.farray.head = 0; pThis->tVars.farray.tail = 0; @@ -483,9 +435,7 @@ static rsRetVal qDestructFixedArray(qqueue_t *pThis) ASSERT(pThis != NULL); queueDrain(pThis); /* discard any remaining queue entries */ - - if(pThis->tVars.farray.pBuf != NULL) - free(pThis->tVars.farray.pBuf); + free(pThis->tVars.farray.pBuf); RETiRet; } @@ -504,76 +454,37 @@ static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) RETiRet; } -static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out) + +static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out) { DEFiRet; ASSERT(pThis != NULL); - *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.head]; + *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.deqhead]; - pThis->tVars.farray.head++; - if (pThis->tVars.farray.head == pThis->iMaxQueueSize) - pThis->tVars.farray.head = 0; + pThis->tVars.farray.deqhead++; + if (pThis->tVars.farray.deqhead == pThis->iMaxQueueSize) + pThis->tVars.farray.deqhead = 0; RETiRet; } -/* -------------------- linked list -------------------- */ - -/* first some generic functions which are also used for the unget linked list */ - -static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) +static rsRetVal qDelFixedArray(qqueue_t *pThis) { DEFiRet; - qLinkedList_t *pEntry; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - pEntry->pNext = NULL; - pEntry->pUsr = pUsr; + ASSERT(pThis != NULL); - if(*ppRoot == NULL) { - *ppRoot = *ppLast = pEntry; - } else { - (*ppLast)->pNext = pEntry; - *ppLast = pEntry; - } + pThis->tVars.farray.head++; + if (pThis->tVars.farray.head == pThis->iMaxQueueSize) + pThis->tVars.farray.head = 0; -finalize_it: RETiRet; } -static inline rsRetVal qqueueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) -{ - DEFiRet; - qLinkedList_t *pEntry; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - ASSERT(ppUsr != NULL); - ASSERT(*ppRoot != NULL); - - pEntry = *ppRoot; - *ppUsr = pEntry->pUsr; - - if(*ppRoot == *ppLast) { - *ppRoot = NULL; - *ppLast = NULL; - } else { - *ppRoot = pEntry->pNext; - } - free(pEntry); - - RETiRet; -} -/* end generic functions which are also used for the unget linked list */ +/* -------------------- linked list -------------------- */ static rsRetVal qConstructLinkedList(qqueue_t *pThis) @@ -582,8 +493,9 @@ static rsRetVal qConstructLinkedList(qqueue_t *pThis) ASSERT(pThis != NULL); - pThis->tVars.linklist.pRoot = 0; - pThis->tVars.linklist.pLast = 0; + pThis->tVars.linklist.pDeqRoot = NULL; + pThis->tVars.linklist.pDelRoot = NULL; + pThis->tVars.linklist.pLast = NULL; qqueueChkIsDA(pThis); @@ -606,54 +518,59 @@ static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) { + qLinkedList_t *pEntry; DEFiRet; - iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); -#if 0 - qLinkedList_t *pEntry; - - ASSERT(pThis != NULL); - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc((pEntry = (qLinkedList_t*) MALLOC(sizeof(qLinkedList_t)))); pEntry->pNext = NULL; pEntry->pUsr = pUsr; - if(pThis->tVars.linklist.pRoot == NULL) { - pThis->tVars.linklist.pRoot = pThis->tVars.linklist.pLast = pEntry; + if(pThis->tVars.linklist.pDelRoot == NULL) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = pEntry; } else { pThis->tVars.linklist.pLast->pNext = pEntry; pThis->tVars.linklist.pLast = pEntry; } + if(pThis->tVars.linklist.pDeqRoot == NULL) { + pThis->tVars.linklist.pDeqRoot = pEntry; + } + finalize_it: -#endif RETiRet; } -static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr) + +static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr) { - DEFiRet; - iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); -#if 0 qLinkedList_t *pEntry; + DEFiRet; - ASSERT(pThis != NULL); - ASSERT(pThis->tVars.linklist.pRoot != NULL); - - pEntry = pThis->tVars.linklist.pRoot; + pEntry = pThis->tVars.linklist.pDeqRoot; + ISOBJ_TYPE_assert(pEntry->pUsr, msg); *ppUsr = pEntry->pUsr; + pThis->tVars.linklist.pDeqRoot = pEntry->pNext; - if(pThis->tVars.linklist.pRoot == pThis->tVars.linklist.pLast) { - pThis->tVars.linklist.pRoot = NULL; - pThis->tVars.linklist.pLast = NULL; + RETiRet; +} + + +static rsRetVal qDelLinkedList(qqueue_t *pThis) +{ + qLinkedList_t *pEntry; + DEFiRet; + + pEntry = pThis->tVars.linklist.pDelRoot; + + if(pThis->tVars.linklist.pDelRoot == pThis->tVars.linklist.pLast) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = NULL; } else { - pThis->tVars.linklist.pRoot = pEntry->pNext; + pThis->tVars.linklist.pDelRoot = pEntry->pNext; } + free(pEntry); -#endif RETiRet; } @@ -667,45 +584,7 @@ qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pTh DEFiRet; ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_TYPE_assert(pThis, qqueue); - CHKiRet(strmSetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); -finalize_it: - RETiRet; -} - - -/* This method checks if we have a QIF file for the current queue (no matter of - * queue mode). Returns RS_RET_OK if we have a QIF file or an error status otherwise. - * rgerhards, 2008-01-15 - */ -static rsRetVal -qqueueHaveQIF(qqueue_t *pThis) -{ - DEFiRet; - uchar pszQIFNam[MAXFNAME]; - size_t lenQIFNam; - struct stat stat_buf; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pszFilePrefix == NULL) - ABORT_FINALIZE(RS_RET_NO_FILEPREFIX); - - /* Construct file name */ - lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", - (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - - /* check if the file exists */ - if(stat((char*) pszQIFNam, &stat_buf) == -1) { - if(errno == ENOENT) { - dbgoprint((obj_t*) pThis, "no .qi file found\n"); - ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); - } else { - dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno); - ABORT_FINALIZE(RS_RET_IO_ERROR); - } - } - /* If we reach this point, we have a .qi file */ - + CHKiRet(strm.SetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); finalize_it: RETiRet; } @@ -722,8 +601,6 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; struct stat stat_buf; - int iUngottenObjs; - obj_t *pUsr; ISOBJ_TYPE_assert(pThis, qqueue); @@ -734,44 +611,41 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* check if the file exists */ if(stat((char*) pszQIFNam, &stat_buf) == -1) { if(errno == ENOENT) { - dbgoprint((obj_t*) pThis, "clean startup, no .qi file found\n"); + DBGOPRINT((obj_t*) pThis, "clean startup, no .qi file found\n"); ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); } else { - dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno); + DBGOPRINT((obj_t*) pThis, "error %d trying to access .qi file\n", errno); ABORT_FINALIZE(RS_RET_IO_ERROR); } } /* If we reach this point, we have a .qi file */ - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_READ)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_READ)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, we try to read the property bag for ourselfs */ CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF)); - /* then the ungotten object queue */ - iUngottenObjs = pThis->iUngottenObjs; - pThis->iUngottenObjs = 0; /* will be incremented when we add objects! */ - - while(iUngottenObjs > 0) { - /* fill the queue from disk */ - CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL)); - qqueueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); - --iUngottenObjs; /* one less */ - } - - /* and now the stream objects (some order as when persisted!) */ + /* then the stream objects (same order as when persisted!) */ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, + CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pRead)); + /* create a duplicate for the read "pointer". + */ + + CHKiRet(strm.Dup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */ + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); + + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDel)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDeq)); /* OK, we could successfully read the file, so we now can request that it be * deleted when we are done with the persisted information. @@ -780,10 +654,10 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); if(iRet != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", + DBGOPRINT((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", iRet); } @@ -815,24 +689,34 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) if(bRestarted == 1) { ; } else { - CHKiRet(strmConstruct(&pThis->tVars.disk.pWrite)); - CHKiRet(strmSetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); - CHKiRet(strmSetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pWrite)); - - CHKiRet(strmConstruct(&pThis->tVars.disk.pRead)); - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); - CHKiRet(strmSetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pRead)); - - - CHKiRet(strmSetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(strmSetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pWrite, pThis->bSyncQueueFiles)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); + + CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); + + CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDel)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pReadDel, pThis->bSyncQueueFiles)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDel, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDel)); + + CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDeq, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDel, pThis->pszFilePrefix, pThis->lenFilePrefix)); } /* now we set (and overwrite in case of a persisted restart) some parameters which @@ -840,8 +724,9 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) * for example file name generation must not be changed as that would break the * ability to read existing queue files. -- rgerhards, 2008-01-12 */ - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDeq, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDel, pThis->iMaxFileSize)); finalize_it: RETiRet; @@ -854,8 +739,12 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) ASSERT(pThis != NULL); - strmDestruct(&pThis->tVars.disk.pWrite); - strmDestruct(&pThis->tVars.disk.pRead); + if(pThis->tVars.disk.pWrite != NULL) + strm.Destruct(&pThis->tVars.disk.pWrite); + if(pThis->tVars.disk.pReadDeq != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDeq); + if(pThis->tVars.disk.pReadDel != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDel); RETiRet; } @@ -867,10 +756,10 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) ASSERT(pThis != NULL); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); CHKiRet((objSerialize(pUsr))(pUsr, pThis->tVars.disk.pWrite)); - CHKiRet(strmFlush(pThis->tVars.disk.pWrite)); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ + CHKiRet(strm.Flush(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ pThis->tVars.disk.sizeOnDisk += nWriteCount; @@ -880,23 +769,37 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) */ objDestruct(pUsr); - dbgoprint((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets\n", + DBGOPRINT((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets\n", nWriteCount, pThis->tVars.disk.sizeOnDisk); finalize_it: RETiRet; } -static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) + +static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr) +{ + DEFiRet; + + CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +static rsRetVal qDelDisk(qqueue_t *pThis) { + obj_t *pDummyObj; /* we need to deserialize it... */ DEFiRet; int64 offsIn; int64 offsOut; - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); - CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pRead, NULL, NULL)); - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsIn)); + CHKiRet(obj.Deserialize(&pDummyObj, (uchar*) "msg", pThis->tVars.disk.pReadDel, NULL, NULL)); + objDestruct(pDummyObj); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsOut)); /* This time it is a bit tricky: we free disk space only upon file deletion. So we need * to keep track of what we have read until we get an out-offset that is lower than the @@ -908,7 +811,7 @@ static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) } else { pThis->tVars.disk.sizeOnDisk -= pThis->tVars.disk.bytesRead; pThis->tVars.disk.bytesRead = offsOut; - dbgoprint((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk); + DBGOPRINT((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk); /* awake possibly waiting enq process */ pthread_cond_signal(&pThis->notFull); /* we hold the mutex while we are in here! */ } @@ -917,6 +820,7 @@ finalize_it: RETiRet; } + /* -------------------- direct (no queueing) -------------------- */ static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { @@ -931,6 +835,8 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { + batch_t singleBatch; + batch_obj_t batchObj; DEFiRet; ASSERT(pThis != NULL); @@ -940,13 +846,21 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) * mode the consumer probably has a lot to convey (which get's lost in the other modes * because they are asynchronous. But direct mode is deliberately synchronous. * rgerhards, 2008-02-12 + * We use our knowledge about the batch_t structure below, but without that, we + * pay a too-large performance toll... -- rgerhards, 2009-04-22 */ - iRet = pThis->pConsumer(pThis->pUsr, pUsr); + batchObj.state = BATCH_STATE_RDY; + batchObj.pUsrp = (obj_t*) pUsr; + singleBatch.nElem = 1; /* there always is only one in direct mode */ + singleBatch.pElem = &batchObj; + iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate); + objDestruct(pUsr); RETiRet; } -static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) + +static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) { return RS_RET_OK; } @@ -955,57 +869,6 @@ static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute_ /* --------------- end type-specific handlers -------------------- */ -/* unget a user pointer that has been dequeued. This functionality is especially important - * for consumer cancel cleanup handlers. To support it, a short list of ungotten user pointers - * is maintened in memory. - * rgerhards, 2008-01-20 - */ -static rsRetVal -qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) -{ - DEFiRet; - DEFVARS_mutexProtection; - - ISOBJ_TYPE_assert(pThis, qqueue); - ISOBJ_assert(pUsr); /* TODO: we aborted right at this place at least 3 times -- race? 2008-02-28, -03-10, -03-15 - The second time I noticed it the queue was in destruction with NO worker threads - running. The pUsr ptr was totally off and provided no clue what it may be pointing - at (except that it looked like the static data pool). Both times, the abort happend - inside an action queue */ - - dbgoprint((obj_t*) pThis, "ungetting user object %s\n", obj.GetName(pUsr)); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); - iRet = qqueueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); - ++pThis->iUngottenObjs; /* indicate one more */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - - RETiRet; -} - - -/* dequeues a user pointer from the ungotten queue. Pointers from there should always be - * dequeued first. - * - * This function must only be called when the mutex is locked! - * - * rgerhards, 2008-01-29 - */ -static rsRetVal -qqueueGetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(ppUsr != NULL); - - iRet = qqueueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); - --pThis->iUngottenObjs; /* indicate one less */ - dbgoprint((obj_t*) pThis, "dequeued ungotten user object %s\n", obj.GetName(*ppUsr)); - - RETiRet; -} - - /* generic code to add a queue entry * We use some specific code to most efficiently support direct mode * queues. This is justified in spite of the gain and the need to do some @@ -1022,7 +885,8 @@ qqueueAdd(qqueue_t *pThis, void *pUsr) if(pThis->qType != QUEUETYPE_DIRECT) { ATOMIC_INC(pThis->iQueueSize); - dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize); + DBGOPRINT((obj_t*) pThis, "entry added, size now log %d, phys %d entries\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); } finalize_it: @@ -1030,12 +894,10 @@ finalize_it: } -/* generic code to remove a queue entry - * rgerhards, 2008-01-29: we must first see if there is any object in the - * ungotten list and, if so, dequeue it first. +/* generic code to dequeue a queue entry */ static rsRetVal -qqueueDel(qqueue_t *pThis, void *pUsr) +qqueueDeq(qqueue_t *pThis, void **ppUsr) { DEFiRet; @@ -1046,224 +908,247 @@ qqueueDel(qqueue_t *pThis, void *pUsr) * If we decrement, however, we may lose a message. But that is better than * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ - if(pThis->iUngottenObjs > 0) { - iRet = qqueueGetUngottenObj(pThis, (obj_t**) pUsr); - } else { - iRet = pThis->qDel(pThis, pUsr); - ATOMIC_DEC(pThis->iQueueSize); - } + iRet = pThis->qDeq(pThis, ppUsr); + ATOMIC_INC(pThis->nLogDeq); - dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n", - iRet, pThis->iQueueSize); +// DBGOPRINT((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", +// getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } -/* This function shuts down all worker threads and waits until they - * have terminated. If they timeout, they are cancelled. Parameters have been set - * before this function is called so that DA queues will be fully persisted to - * disk (if configured to do so). - * rgerhards, 2008-01-24 - * Please note that this function shuts down BOTH the parent AND the child queue - * in DA case. This is necessary because their timeouts are tightly coupled. Most - * importantly, the timeouts would be applied twice (or logic be extremely - * complex) if each would have its own shutdown. The function does not self check - * this condition - the caller must make sure it is not called with a parent. +/* Try to shut down regular and DA queue workers, within the queue timeout + * period. That means processing continues as usual. This is the expected + * usual case, where during shutdown those messages remaining are being + * processed. At this point, it is acceptable that the queue can not be + * fully depleted, that case is handled in the next step. During this phase, + * we first shut down the main queue DA worker to prevent new data to arrive + * at the DA queue, and then we ask the regular workers of both the Regular + * and DA queue to try complete processing. + * rgerhards, 2009-10-14 */ -static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) +static inline rsRetVal +tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; struct timespec tTimeout; rsRetVal iRetLocal; + DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ - dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); + if(pThis->bIsDA) { + /* We need to lock the mutex, as otherwise we may have a race that prevents + * us from awaking the DA worker. */ + d_pthread_mutex_lock(pThis->mut); - /* we reduce the low water mark in any case. This is not absolutely necessary, but - * it is useful because we enable DA mode at several spots below and so we do not need - * to think about the low water mark each time. - */ - pThis->iHighWtrMrk = 1; /* if we do not do this, the DA queue will not stop! */ - pThis->iLowWtrMrk = 0; + /* tell regular queue DA worker to stop shuffling messages to DA queue... */ + pThis->pqDA->bEnqOnly = 1; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE); + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); + DBGOPRINT((obj_t*) pThis, "awoke DA worker, told it to shut down.\n"); - /* first try to shutdown the queue within the regular shutdown period */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - if(pThis->bRunsDA) { - /* We may have waited on the low water mark. As it may have changed, we - * see if we reactivate the worker. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); - } + /* also tell the DA queue worker to shut down, so that it already knows... */ + wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN); + wtpAdviseMaxWorkers(pThis->pqDA->pWtpReg, 1); /* awake its lone worker */ + DBGOPRINT((obj_t*) pThis, "awoke DA queue regular worker, told it to shut down when done.\n"); + + d_pthread_mutex_unlock(pThis->mut); } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - /* Now wait for the queue's workers to shut down. Note that we run into the code even if we just found - * out there are no active workers - that doesn't matter: the wtp knows about that and so will - * return immediately. - * We do not yet care about the DA worker - that will be handled down later in the process. - * Note that we must not request shutdown right now - that may introduce a race: if the regular queue - * still runs DA assisted and the DA worker gets scheduled first, it will terminate itself (if the DA - * queue happens to be empty at that instant). Then the regular worker enqueues messages, what will lead - * to a restart of the worker. Of course, everything will continue to run, but in a bit sub-optimal way - * (from a performance point of view). So we don't do anything right now. The DA queue will continue to - * process messages and shutdown itself in any case if there is nothing to do. So we don't loose anything - * by not requesting shutdown now. - * rgerhards, 2008-01-25 - */ + /* first calculate absolute timeout - we need the absolute value here, because we need to coordinate * shutdown of both the regular and DA queue on *the same* timeout. */ timeoutComp(&tTimeout, pThis->toQShutdown); - dbgoprint((obj_t*) pThis, "trying shutdown of regular workers\n"); + DBGOPRINT((obj_t*) pThis, "trying shutdown of regular workers\n"); iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN, &tTimeout); if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n"); + DBGOPRINT((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n"); } else { - /* OK, the regular queue is now shut down. So we can now wait for the DA queue (if running DA) */ - dbgoprint((obj_t*) pThis, "regular queue workers shut down.\n"); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(pThis->bRunsDA) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", - qqueueGetID(pThis->pqDA)); - /* we use the same absolute timeout as above, so we do not use more than the configured - * timeout interval! - */ - dbgoprint((obj_t*) pThis, "trying shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "shutdown timed out on DA queue (this is OK)\n"); - } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - } + DBGOPRINT((obj_t*) pThis, "regular queue workers shut down.\n"); } - /* when we reach this point, both queues are either empty or the regular queue shutdown timeout - * has expired. Now we need to check if we are configured to not loose messages. If so, we need - * to persist the queue to disk (this is only possible if the queue is DA-enabled). We must also - * set the primary queue to SHUTDOWN_IMMEDIATE, as it shall now terminate as soon as its consumer - * is done. This is especially important as we otherwise may interfere with queue order while the - * DA consumer is running. -- rgerhards, 2008-01-27 - * Note: there was a note that we should not wait eternally on the DA worker if we run in - * enqueue-only note. I have reviewed the code and think there is no need for this check. Howerver, - * I'd like to keep this note in here should we happen to run into some related trouble. - * rgerhards, 2008-01-28 - */ - wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); /* set primary queue to shutdown only */ - - /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ - if(pThis->bRunsDA) - qqueueWaitDAModeInitialized(pThis); - - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - /* optimize parameters for shutdown of DA-enabled queues */ - if(pThis->bIsDA && qqueueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { - /* switch to enqueue-only mode so that no more actions happen */ - if(pThis->bRunsDA == 0) { - qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ + /* OK, the worker for the regular queue is processed, on the the DA queue regular worker. */ + if(pThis->pqDA != NULL) { + DBGOPRINT((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", + qqueueGetID(pThis->pqDA)); + /* we use the same absolute timeout as above, so we do not use more than the configured + * timeout interval! + */ + DBGOPRINT((obj_t*) pThis, "trying shutdown of regular worker of DA queue\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "shutdown timed out on DA queue worker (this is OK)\n"); } else { - /* TODO: RACE: we may reach this point when the DA worker has been initialized (state 1) - * but is not yet running (state 2). In this case, pThis->pqDA is NULL! rgerhards, 2008-02-27 - */ - qqueueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ + DBGOPRINT((obj_t*) pThis, "DA queue worker shut down.\n"); } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - /* make sure we do not timeout before we are done */ - dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, eternal timeout set\n"); - timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); - /* and run the primary queue's DA worker to drain the queue */ - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " - "continuing, but results are unpredictable\n", iRetLocal); - } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); } - /* now the primary queue is either empty, persisted to disk - or set to loose messages. So we - * can now request immediate shutdown of any remaining workers. Note that if bSaveOnShutdown was set, - * the queue is now empty. If regular workers are still running, and try to pull the next message, - * they will automatically terminate as there no longer is any message left to process. - */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - timeoutComp(&tTimeout, pThis->toActShutdown); - if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of regular workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } - /* we need to re-aquire the mutex for the next check in this case! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ + RETiRet; +} + + +/* Try to shut down regular and DA queue workers, within the action timeout + * period. This aborts processing, but at the end of the current action, in + * a well-defined manner. During this phase, we terminate all three worker + * pools, including the regular queue DA worker if it not yet has terminated. + * Not finishing processing all messages is OK (and expected) at this stage + * (they may be preserved later, depending * on bSaveOnShutdown setting). + * rgerhards, 2009-10-14 + */ +static rsRetVal +tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) +{ + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + +RUNLOG_STR("trying to shutdown workers within Action Timeout"); + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + /* instruct workers to finish ASAP, even if still work exists */ + pThis->bEnqOnly = 1; + pThis->bShutdownImmediate = 1; + /* now DA queue */ + if(pThis->bIsDA) { + pThis->pqDA->bEnqOnly = 1; + pThis->pqDA->bShutdownImmediate = 1; + } + +// TODO: make sure we have at minimum a 10ms timeout - workers deserve a chance... + /* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */ + timeoutComp(&tTimeout, pThis->toActShutdown); + DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of regular workers (if any)\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " + "triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " + "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); + } + + if(pThis->pqDA != NULL) { + /* and now the same for the DA queue */ + DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of DA queue workers\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable " + "and triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA " + "queue in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); } - if(pThis->bIsDA && wtpGetCurNumWrkr(pThis->pWtpDA, LOCK_MUTEX) > 0) { - /* and now the same for the DA queue */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } + + /* and now we need to terminate the DA worker itself. We always grant it a 100ms timeout, + * which should be sufficient and usually not be required (it is expected to have finished + * long before while we were processing the queue timeout in shutdown phase 1). + * rgerhards, 2009-10-14 + */ + timeoutComp(&tTimeout, 100); + DBGOPRINT((obj_t*) pThis, "trying regular shutdown of main queue DA worker pool\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "shutdown timed out on main queue DA worker pool " + "(this is not good, but probably OK)\n"); } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + DBGOPRINT((obj_t*) pThis, "main queue DA worker pool shut down.\n"); } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); } + RETiRet; +} + + +/* This function cancels all remaining regular workers for both the main and the DA + * queue. + * rgerhards, 2009-05-29 + */ +static rsRetVal +cancelWorkers(qqueue_t *pThis) +{ + rsRetVal iRetLocal; + DEFiRet; + /* Now queue workers should have terminated. If not, we need to cancel them as we have applied * all timeout setting. If any worker in any queue still executes, its consumer is possibly - * long-running and cancelling is the only way to get rid of it. Note that the - * cancellation handler will probably re-queue a user pointer, so the queue's enqueue - * function is still needed (what is no problem as we do not yet destroy the queue - but I - * thought it's a good idea to mention that fact). -- rgerhards, 2008-01-25 + * long-running and cancelling is the only way to get rid of it. */ - dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n"); + DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n"); iRetLocal = wtpCancelAll(pThis->pWtpReg); /* returns immediately if all threads already have terminated */ if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker " + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker " "threads, continuing, but results are unpredictable\n", iRetLocal); } - - /* TODO: think: do we really need to do this here? Can't it happen on DA queue destruction? If we - * disable it, we get an assertion... I think this is OK, as we need to have a certain order and - * canceling the DA workers here ensures that order. But in any instant, we may have a look at this - * code after we have reaced the milestone. -- rgerhards, 2008-01-27 - */ /* ... and now the DA queue, if it exists (should always be after the primary one) */ if(pThis->pqDA != NULL) { - dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n"); + DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n"); iRetLocal = wtpCancelAll(pThis->pqDA->pWtpReg); /* returns immediately if all threads already have terminated */ if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker " + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker " "threads, continuing, but results are unpredictable\n", iRetLocal); } + + /* finally, we cancel the main queue's DA worker pool, if it still is running. It may be + * restarted later to persist the queue. But we stop it, because otherwise we get into + * big trouble when resetting the logical dequeue pointer. This operation can only be + * done when *no* worker is running. So time for a shutdown... -- rgerhards, 2009-05-28 + */ + DBGOPRINT((obj_t*) pThis, "checking to see if main queue DA worker pool needs to be cancelled\n"); + iRetLocal = wtpCancelAll(pThis->pWtpDA); /* returns immediately if all threads already have terminated */ } + RETiRet; +} + + +/* This function shuts down all worker threads and waits until they + * have terminated. If they timeout, they are cancelled. + * rgerhards, 2008-01-24 + * Please note that this function shuts down BOTH the parent AND the child queue + * in DA case. This is necessary because their timeouts are tightly coupled. Most + * importantly, the timeouts would be applied twice (or logic be extremely + * complex) if each would have its own shutdown. The function does not self check + * this condition - the caller must make sure it is not called with a parent. + * rgerhards, 2009-05-26: we do NO longer persist the queue here if bSaveOnShutdown + * is set. This must be handled by the caller. Not doing that cleans up the queue + * shutdown considerably. Also, older engines had a potential hang condition when + * the DA queue was already started and the DA worker configured for infinite + * retries and the action was during retry processing. This was a design issue, + * which is solved as of now. Note that the shutdown now may take a little bit + * longer, because we no longer can persist the queue in parallel to waiting + * on worker timeouts. + */ +static rsRetVal +ShutdownWorkers(qqueue_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + DBGOPRINT((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); + + CHKiRet(tryShutdownWorkersWithinQueueTimeout(pThis)); + + if(getPhysicalQueueSize(pThis) > 0) { + CHKiRet(tryShutdownWorkersWithinActionTimeout(pThis)); + } + + CHKiRet(cancelWorkers(pThis)); + /* ... finally ... all worker threads have terminated :-) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers - * may still be running. + * may still be running. Note that the main queue's DA worker may still be running. */ - dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", qqueueGetOverallQueueSize(pThis)); + DBGOPRINT((obj_t*) pThis, "worker threads terminated, remaining queue size log %d, phys %d.\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); +finalize_it: RETiRet; } @@ -1275,7 +1160,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * to modify some parameters before the queue is actually started. */ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)) + int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*,int*)) { DEFiRet; qqueue_t *pThis; @@ -1284,13 +1169,10 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread ASSERT(pConsumer != NULL); ASSERT(iWorkerThreads >= 0); - if((pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))); /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -1298,13 +1180,15 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->iFullDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 3; /* default 97% */ pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */ - pThis->lenSpoolDir = strlen((char*)pThis->pszSpoolDir); + pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; + pThis->nLogDeq = 0; pThis->iMaxQueueSize = iMaxQueueSize; pThis->pConsumer = pConsumer; pThis->iNumWorkerThreads = iWorkerThreads; pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */ + pThis->iDeqBatchSize = 8; /* conservative default, should still provide good performance */ pThis->pszFilePrefix = NULL; pThis->qType = qType; @@ -1315,18 +1199,21 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qConstruct = qConstructFixedArray; pThis->qDestruct = qDestructFixedArray; pThis->qAdd = qAddFixedArray; + pThis->qDeq = qDeqFixedArray; pThis->qDel = qDelFixedArray; break; case QUEUETYPE_LINKEDLIST: pThis->qConstruct = qConstructLinkedList; pThis->qDestruct = qDestructLinkedList; pThis->qAdd = qAddLinkedList; - pThis->qDel = (rsRetVal (*)(qqueue_t*,void**)) qDelLinkedList; + pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList; + pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; pThis->qDestruct = qDestructDisk; pThis->qAdd = qAddDisk; + pThis->qDeq = qDeqDisk; pThis->qDel = qDelDisk; /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ @@ -1345,40 +1232,10 @@ finalize_it: } -/* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. - * Params: - * arg1 - user pointer (in this case a qqueue_t) - * arg2 - user data pointer (in this case a queue data element, any object [queue's pUsr ptr!]) - * Note that arg2 may be NULL, in which case no dequeued but unprocessed pUsr exists! - * rgerhards, 2008-01-16 - */ -static rsRetVal -qqueueConsumerCancelCleanup(void *arg1, void *arg2) -{ - DEFiRet; - - qqueue_t *pThis = (qqueue_t*) arg1; - obj_t *pUsr = (obj_t*) arg2; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pUsr != NULL) { - /* make sure the data element is not lost */ - dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n"); - CHKiRet(qqueueUngetObj(pThis, pUsr, LOCK_MUTEX)); - } - -finalize_it: - RETiRet; -} - - - /* This function checks if the provided message shall be discarded and does so, if needed. * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to * provide real-time creation of spool files. - * Note: cached copies of iQueueSize and bRunsDA are provided so that no mutex locks are required. + * Note: cached copies of iQueueSize is provided so that no mutex locks are required. * The caller must have obtained them while the mutex was locked. Of course, these values may no * longer be current, but that is OK for the discard check. At worst, the message is either processed * or discarded when it should not have been. As discarding is in itself somewhat racy and erratic, @@ -1388,7 +1245,7 @@ finalize_it: * the return state! * rgerhards, 2008-01-24 */ -static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) +static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, void *pUsr) { DEFiRet; rsRetVal iRetLocal; @@ -1397,15 +1254,15 @@ static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, voi ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); - if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk && bRunsDA == 0) { + if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk) { iRetLocal = objGetSeverity(pUsr, &iSeverity); if(iRetLocal == RS_RET_OK && iSeverity >= pThis->iDiscardSeverity) { - dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", + DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", iQueueSize, iSeverity); objDestruct(pUsr); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } else { - dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg " + DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg " "(iRet: %d, severity %d)\n", iQueueSize, iRetLocal, iSeverity); } } @@ -1415,78 +1272,207 @@ finalize_it: } -/* dequeue the queued object for the queue consumers. - * rgerhards, 2008-10-21 +/* Finally remove n elements from the queue store. */ -static rsRetVal -qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +static inline rsRetVal +DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) +{ + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + /* now send delete request to storage driver */ + for(i = 0 ; i < nElem ; ++i) { + pThis->qDel(pThis); + } + + /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ + ATOMIC_SUB(pThis->iQueueSize, nElem); + ATOMIC_SUB(pThis->nLogDeq, nElem); +dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + ++pThis->deqIDDel; /* one more batch dequeued */ + + RETiRet; +} + + +/* remove messages from the physical queue store that are fully processed. This is + * controlled via the to-delete list. + */ +static inline rsRetVal +DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) { + toDeleteLst_t *pTdl; + qDeqID deqIDDel; DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + pTdl = tdlPeek(pThis); /* get current head element */ + if(pTdl == NULL) { /* to-delete list empty */ + DoDeleteBatchFromQStore(pThis, pBatch->nElem); + } else if(pBatch->deqID == pThis->deqIDDel) { + deqIDDel = pThis->deqIDDel; + pTdl = tdlPeek(pThis); + while(pTdl != NULL && deqIDDel == pTdl->deqID) { + DoDeleteBatchFromQStore(pThis, pTdl->nElemDeq); + tdlPop(pThis); + ++deqIDDel; + pTdl = tdlPeek(pThis); + } + /* old entries deleted, now delete current ones... */ + DoDeleteBatchFromQStore(pThis, pBatch->nElem); + } else { + /* can not delete, insert into to-delete list */ + dbgprintf("not at head of to-delete list, enqueue %d\n", (int) pBatch->deqID); + CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem)); + } + +finalize_it: + RETiRet; +} + + +/* Delete a batch of processed user objects from the queue, which includes + * destructing the objects themself. Any entries not marked as finally + * processed are enqueued again. The new enqueue is necessary because we have a + * rgerhards, 2009-05-13 + */ +static inline rsRetVal +DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) +{ + int i; void *pUsr; + int nEnqueued = 0; + rsRetVal localRet; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + for(i = 0 ; i < pBatch->nElem ; ++i) { + pUsr = pBatch->pElem[i].pUsrp; + if( pBatch->pElem[i].state == BATCH_STATE_RDY + || pBatch->pElem[i].state == BATCH_STATE_SUB) { + localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*) pUsr)); + ++nEnqueued; + if(localRet != RS_RET_OK) { + DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet); + } + } + objDestruct(pUsr); + } + + dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued); + + if(nEnqueued > 0) + qqueueChkPersist(pThis, nEnqueued); + + iRet = DeleteBatchFromQStore(pThis, pBatch); + + pBatch->nElem = pBatch->nElemDeq = 0; /* reset batch */ + + RETiRet; +} + + +/* dequeue as many user pointers as are available, until we hit the configured + * upper limit of pointers. Note that this function also deletes all processed + * objects from the previous batch. However, it is perfectly valid that the + * previous batch contained NO objects at all. For example, this happens + * immediately after system startup or when a queue was exhausted and the queue + * worker needed to wait for new data. + * This must only be called when the queue mutex is LOOKED, otherwise serious + * malfunction will happen. + */ +static inline rsRetVal +DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSize) +{ + int nDequeued; + int nDiscarded; + int nDeleted; int iQueueSize; - int bRunsDA; /* cache for early mutex release */ - - /* dequeue element (still protected from mutex) */ - iRet = qqueueDel(pThis, &pUsr); - qqueueChkPersist(pThis); - iQueueSize = qqueueGetOverallQueueSize(pThis); /* cache this for after mutex release */ - bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ - - /* We now need to save the user pointer for the cancel cleanup handler, BUT ONLY - * if we could successfully obtain a user pointer. Otherwise, we would bring the - * cancel cleanup handler into big troubles (and we did ;)). Note that we can - * NOT set the variable further below, as this may lead to an object leak. We - * may get cancelled before we reach that part of the code, so the only - * solution is to do it here. -- rgerhards, 2008-02-27 - */ - if(iRet == RS_RET_OK) { - pWti->pUsrp = pUsr; + void *pUsr; + rsRetVal localRet; + DEFiRet; + + nDeleted = pWti->batch.nElemDeq; + DeleteProcessedBatch(pThis, &pWti->batch); + + nDequeued = nDiscarded = 0; + while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { + CHKiRet(qqueueDeq(pThis, &pUsr)); + + /* check if we should discard this element */ + localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr); + if(localRet == RS_RET_QUEUE_FULL) { + ++nDiscarded; + continue; + } else if(localRet != RS_RET_OK) { + ABORT_FINALIZE(localRet); + } + + /* all well, use this element */ + pWti->batch.pElem[nDequeued].pUsrp = pUsr; + pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY; + ++nDequeued; } + /* it is sufficient to persist only when the bulk of work is done */ + qqueueChkPersist(pThis, nDequeued+nDiscarded+nDeleted); + + pWti->batch.nElem = nDequeued; + pWti->batch.nElemDeq = nDequeued + nDiscarded; + pWti->batch.deqID = getNextDeqID(pThis); + *piRemainingQueueSize = iQueueSize; + +finalize_it: + RETiRet; +} + + +/* dequeue the queued object for the queue consumers. + * rgerhards, 2008-10-21 + * I made a radical change - we now dequeue multiple elements, and store these objects in + * an array of user pointers. We expect that this increases performance. + * rgerhards, 2009-04-22 + */ +static rsRetVal +DequeueConsumable(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + int iQueueSize = 0; /* keep the compiler happy... */ + + /* dequeue element batch (still protected from mutex) */ + iRet = DequeueConsumableElements(pThis, pWti, &iQueueSize); + /* awake some flow-controlled sources if we can do this right now */ /* TODO: this could be done better from a performance point of view -- do it only if * we have someone waiting for the condition (or only when we hit the watermark right * on the nail [exact value]) -- rgerhards, 2008-03-14 + * now that we dequeue batches of pointers, this is much less an issue... + * rgerhards, 2009-04-22 */ - if(iQueueSize < pThis->iFullDlyMrk) { + if(iQueueSize < pThis->iFullDlyMrk / 2) { pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk); } - if(iQueueSize < pThis->iLightDlyMrk) { + if(iQueueSize < pThis->iLightDlyMrk / 2) { pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk); } - /* rgerhards, 2008-09-30: I reversed the order of cond_signal und mutex_unlock - * as of the pthreads recommendation on predictable scheduling behaviour. I don't see - * any problems caused by this, but I add this comment in case some will be seen - * in the next time. - */ + // TODO: MULTI: check physical queue size? pthread_cond_signal(&pThis->notFull); - d_pthread_mutex_unlock(pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); /* WE ARE NO LONGER PROTECTED BY THE MUTEX */ - /* do actual processing (the lengthy part, runs in parallel) - * If we had a problem while dequeing, we do not call the consumer, - * but we otherwise ignore it. This is in the hopes that it will be - * self-healing. However, this is really not a good thing. - * rgerhards, 2008-01-03 - */ - if(iRet != RS_RET_OK) - FINALIZE; - - /* we are running in normal, non-disk-assisted mode do a quick check if we need to drain the queue. - * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to - * provide real-time creation of spool files. - * Note: It is OK to use the cached iQueueSize here, because it does not hurt if it is slightly wrong. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); - -finalize_it: if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) { - dbgoprint((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things " + DBGOPRINT((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things " "may happen\n", iRet); } + RETiRet; } @@ -1529,7 +1515,7 @@ finalize_it: * but you get the idea from the code above. */ static rsRetVal -qqueueRateLimiter(qqueue_t *pThis) +RateLimiter(qqueue_t *pThis) { DEFiRet; int iDelay; @@ -1542,7 +1528,7 @@ qqueueRateLimiter(qqueue_t *pThis) iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ /* time calls are expensive, so only do them when needed */ - time(&tCurr); + datetime.GetTime(&tCurr); localtime_r(&tCurr, &m); iHrCurr = m.tm_hour; @@ -1578,7 +1564,7 @@ qqueueRateLimiter(qqueue_t *pThis) } if(iDelay > 0) { - dbgoprint((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); + DBGOPRINT((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); srSleep(iDelay, 0); } @@ -1586,37 +1572,95 @@ qqueueRateLimiter(qqueue_t *pThis) } +/* This dequeues the next batch. Note that this function must not be + * cancelled, else it will leave back an inconsistent state. + * rgerhards, 2009-05-20 + */ +static inline rsRetVal +DequeueForConsumer(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); + + CHKiRet(DequeueConsumable(pThis, pWti)); + + if(pWti->batch.nElem == 0) + ABORT_FINALIZE(RS_RET_IDLE); + + +finalize_it: + RETiRet; +} + + +/* This is called when a batch is processed and the worker does not + * ask for another batch (e.g. because it is to be terminated) + * rgerhards, 2009-05-27 + */ +static rsRetVal +batchProcessed(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); + + DeleteProcessedBatch(pThis, &pWti->batch); + qqueueChkPersist(pThis, pWti->batch.nElemDeq); + + RETiRet; +} + /* This is the queue consumer in the regular (non-DA) case. It is * protected by the queue mutex, but MUST release it as soon as possible. * rgerhards, 2008-01-21 */ static rsRetVal -qqueueConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerReg(qqueue_t *pThis, wti_t *pWti) { + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->pUsrp)); + CHKiRet(DequeueForConsumer(pThis, pWti)); + + /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ + d_pthread_mutex_unlock(pThis->mut); + + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + + CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch, &pThis->bShutdownImmediate)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 */ +//TODO: MULTIQUEUE: the following setting is no longer correct - need to think about how to do that... if(pThis->iDeqSlowdown) { - dbgoprint((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n", + DBGOPRINT((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n", pThis->iDeqSlowdown); srSleep(pThis->iDeqSlowdown / 1000000, pThis->iDeqSlowdown % 1000000); } + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + + /* now we are done, but need to re-aquire the mutex */ + d_pthread_mutex_lock(pThis->mut); + finalize_it: + dbgprintf("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } -/* This is a special consumer to feed the disk-queue in disk-assited mode. +/* This is a special consumer to feed the disk-queue in disk-assisted mode. * When active, our own queue more or less acts as a memory buffer to the disk. * So this consumer just needs to drain the memory queue and submit entries * to the disk queue. The disk queue will then call the actual consumer from @@ -1626,151 +1670,93 @@ finalize_it: * rgerhards, 2008-01-14 */ static rsRetVal -qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerDA(qqueue_t *pThis, wti_t *pWti) { + int i; + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); + CHKiRet(DequeueForConsumer(pThis, pWti)); + + /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ + d_pthread_mutex_unlock(pThis->mut); + + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + + /* iterate over returned results and enqueue them in DA queue */ + for(i = 0 ; i < pWti->batch.nElem && !pThis->bShutdownImmediate ; i++) { + /* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs + * the message. So far, we simply assume we always have msg_t, what currently is always the case. + * rgerhards, 2009-05-28 + */ +dbgprintf("DA consumer pushes msg '%s'\n", ((msg_t*)(pWti->batch.pElem[i].pUsrp))->pszRawMsg); + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp)))); + pWti->batch.pElem[i].state = BATCH_STATE_COMM; /* commited to other queue! */ + } + + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + + /* now we are done, but need to re-aquire the mutex */ + d_pthread_mutex_lock(pThis->mut); finalize_it: - dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); + DBGOPRINT((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); RETiRet; } /* must only be called when the queue mutex is locked, else results * are not stable! - * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the DA worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue */ -static int +static rsRetVal qqueueChkStopWrkrDA(qqueue_t *pThis) { - /* if our queue is in destruction, we drain to the DA queue and so we shall not terminate - * until we have done so. - */ - int bStopWrkr; - - BEGINfunc + DEFiRet; if(pThis->bEnqOnly) { - bStopWrkr = 1; - } else { - if(pThis->bRunsDA) { - ASSERT(pThis->pqDA != NULL); - if( pThis->pqDA->bEnqOnly - && pThis->pqDA->sizeOnDiskMax > 0 - && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { - /* this queue can never grow, so we can give up... */ - bStopWrkr = 1; - } else if(qqueueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { - bStopWrkr = 1; - } else { - bStopWrkr = 0; - } - } else { - bStopWrkr = 1; - } + iRet = RS_RET_TERMINATE_WHEN_IDLE; } - ENDfunc - return bStopWrkr; + RETiRet; } /* must only be called when the queue mutex is locked, else results * are not stable! * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the regular worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue - */ -static int -qqueueChkStopWrkrReg(qqueue_t *pThis) -{ - return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0); -} - - -/* must only be called when the queue mutex is locked, else results - * are not stable! DA queue version - */ -static int -qqueueIsIdleDA(qqueue_t *pThis) -{ - /* remember: iQueueSize is the DA queue size, not the main queue! */ - /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); -} -/* must only be called when the queue mutex is locked, else results - * are not stable! Regular queue version - */ -static int -qqueueIsIdleReg(qqueue_t *pThis) -{ -#if 0 /* enable for performance testing */ - int ret; - ret = qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); - if(ret) fprintf(stderr, "queue is idle\n"); - return ret; -#else - /* regular code! */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); -#endif -} - - -/* This function is called when a worker thread for the regular queue is shut down. - * If we are the primary queue, this is not really interesting to us. If, however, - * we are the DA (child) queue, that means the DA queue is empty. In that case, we - * need to signal the parent queue's DA worker, so that it can terminate DA mode. - * rgerhards, 2008-01-26 - * rgerhards, 2008-02-27: HOWEVER, in a shutdown condition, it may be that the parent's worker thread pool - * has already been terminated and destructed. This *is* a legal condition and happens - * from time to time in practice. So we need to signal only if there still is a - * parent DA worker queue. Please keep in mind that the the parent's DA worker - * pool is DIFFERENT from our (DA queue) regular worker pool. So when the parent's - * pWtpDA is destructed, there can still be some of our (DAq/wtp) threads be running. - * I am telling this, because I, too, always get confused by those... + * we can terminate. Version for the regular worker thread. */ static rsRetVal -qqueueRegOnWrkrShutdown(qqueue_t *pThis) +ChkStopWrkrReg(qqueue_t *pThis) { DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pqParent != NULL) { - pThis->pqParent->bChildIsDone = 1; /* indicate we are done */ - if(pThis->pqParent->pWtpDA != NULL) { /* see comment in function header from 2008-02-27 */ - wtpAdviseMaxWorkers(pThis->pqParent->pWtpDA, 1); /* reactivate DA worker (always 1) */ - } + if(pThis->bEnqOnly) { + iRet = RS_RET_TERMINATE_NOW; + } else if(pThis->pqParent != NULL) { + iRet = RS_RET_TERMINATE_WHEN_IDLE; } RETiRet; } -/* The following function is called when a regular queue worker starts up. We need this - * hook to indicate in the parent queue (if we are a child) that we are not done yet. +/* return the configured "deq max at once" interval + * rgerhards, 2009-04-22 */ static rsRetVal -qqueueRegOnWrkrStartup(qqueue_t *pThis) +GetDeqBatchSize(qqueue_t *pThis, int *pVal) { DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pqParent != NULL) { - pThis->pqParent->bChildIsDone = 0; - } - + assert(pVal != NULL); + *pVal = pThis->iDeqBatchSize; +if(pThis->pqParent != NULL) // TODO: check why we actually do this! + *pVal = 16; RETiRet; } @@ -1778,11 +1764,10 @@ qqueueRegOnWrkrStartup(qqueue_t *pThis) /* start up the queue - it must have been constructed and parameters defined * before. */ -rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ +rsRetVal +qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; - rsRetVal iRetLocal; - int bInitialized = 0; /* is queue already initialized? */ uchar pszBuf[64]; size_t lenBuf; @@ -1796,11 +1781,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ * influenced by properties which might have been set after queueConstruct () */ if(pThis->pqParent == NULL) { - pThis->mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + pThis->mut = (pthread_mutex_t *) MALLOC (sizeof (pthread_mutex_t)); pthread_mutex_init(pThis->mut, NULL); } else { /* child queue, we need to use parent's mutex */ - dbgoprint((obj_t*) pThis, "I am a child\n"); + DBGOPRINT((obj_t*) pThis, "I am a child\n"); pThis->mut = pThis->pqParent->mut; } @@ -1814,28 +1799,26 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* call type-specific constructor */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ - dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d, " - "full delay %d, light delay %d starting\n", + DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, " + "full delay %d, light delay %d, deq batch size %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, - qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, - pThis->iFullDlyMrk, pThis->iLightDlyMrk); + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), + pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, + pThis->iDeqBatchSize); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ - /* create worker thread pools for regular operation. The DA pool is created on an as-needed - * basis, which potentially means never under most circumstances. + /* create worker thread pools for regular and DA operation. */ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpReg)); CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); - CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRateLimiter)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrReg)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleReg)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerReg)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))qqueueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrStartup)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrShutdown)); + CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerReg)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads)); @@ -1843,27 +1826,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpUsr (pThis->pWtpReg, pThis)); CHKiRet(wtpConstructFinalize (pThis->pWtpReg)); - /* initialize worker thread instances */ - if(pThis->bIsDA) { - /* If we are disk-assisted, we need to check if there is a QIF file - * which we need to load. -- rgerhards, 2008-01-15 - */ - iRetLocal = qqueueHaveQIF(pThis); - if(iRetLocal == RS_RET_OK) { - dbgoprint((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n"); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ - bInitialized = 1; /* we are done */ - } else { - /* TODO: use logerror? -- rgerhards, 2008-01-16 */ - dbgoprint((obj_t*) pThis, "error %d trying to access on-disk queue files, starting without them. " - "Some data may be lost\n", iRetLocal); - } - } + /* set up DA system if we have a disk-assisted queue */ + if(pThis->bIsDA) + InitDA(pThis, LOCK_MUTEX); /* initiate DA mode */ - if(!bInitialized) { - dbgoprint((obj_t*) pThis, "queue starts up without (loading) any DA disk state (this is normal for the DA " - "queue itself!)\n"); - } + DBGOPRINT((obj_t*) pThis, "queue finished initialization\n"); /* if the queue already contains data, we need to start the correct number of worker threads. This can be * the case when a disk queue has been loaded. If we did not start it here, it would never start. @@ -1889,12 +1856,11 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) strm_t *psQIF = NULL; /* Queue Info File */ uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; - obj_t *pUsr; ASSERT(pThis != NULL); if(pThis->qType != QUEUETYPE_DISK) { - if(qqueueGetOverallQueueSize(pThis) > 0) { + if(getPhysicalQueueSize(pThis) > 0) { /* This error code is OK, but we will probably not implement this any time * The reason is that persistence happens via DA queues. But I would like to * leave the code as is, as we so have a hook in case we need one. @@ -1905,28 +1871,29 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) FINALIZE; /* if the queue is empty, we are happy and done... */ } - dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", qqueueGetOverallQueueSize(pThis)); + DBGOPRINT((obj_t*) pThis, "persisting queue to disk, %d entries...\n", getPhysicalQueueSize(pThis)); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - if((bIsCheckpoint != QUEUE_CHECKPOINT) && (qqueueGetOverallQueueSize(pThis) == 0)) { + if((bIsCheckpoint != QUEUE_CHECKPOINT) && (getPhysicalQueueSize(pThis) == 0)) { if(pThis->bNeedDelQIF) { unlink((char*)pszQIFNam); pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + if(pThis->tVars.disk.pReadDel != NULL) /* may be NULL if we had a startup failure! */ + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); FINALIZE; /* nothing left to do, so be happy */ } - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_WRITE)); - CHKiRet(strmSetiAddtlOpenFlags(psQIF, O_TRUNC)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC)); + CHKiRet(strm.SetbSync(psQIF, pThis->bSyncQueueFiles)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, write the property bag for ourselfs * And, surprisingly enough, we currently need to persist only the size of the @@ -1936,29 +1903,19 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) */ CHKiRet(obj.BeginSerializePropBag(psQIF, (obj_t*) pThis)); objSerializeSCALAR(psQIF, iQueueSize, INT); - objSerializeSCALAR(psQIF, iUngottenObjs, INT); objSerializeSCALAR(psQIF, tVars.disk.sizeOnDisk, INT64); objSerializeSCALAR(psQIF, tVars.disk.bytesRead, INT64); CHKiRet(obj.EndSerialize(psQIF)); - /* now we must persist all objects on the ungotten queue - they can not go to - * to the regular files. -- rgerhards, 2008-01-29 - */ - while(pThis->iUngottenObjs > 0) { - CHKiRet(qqueueGetUngottenObj(pThis, &pUsr)); - CHKiRet((objSerialize(pUsr))(pUsr, psQIF)); - objDestruct(pUsr); - } - /* now persist the stream info */ - CHKiRet(strmSerialize(pThis->tVars.disk.pWrite, psQIF)); - CHKiRet(strmSerialize(pThis->tVars.disk.pRead, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pReadDel, psQIF)); /* tell the input file object that it must not delete the file on close if the queue * is non-empty - but only if we are not during a simple checkpoint */ if(bIsCheckpoint != QUEUE_CHECKPOINT) { - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 0)); } /* we have persisted the queue object. So whenever it comes to an empty queue, @@ -1968,28 +1925,75 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); RETiRet; } /* check if we need to persist the current queue info. If an - * error occurs, thus should be ignored by caller (but we still + * error occurs, this should be ignored by caller (but we still * abide to our regular call interface)... * rgerhards, 2008-01-13 + * nUpdates is the number of updates since the last call to this function. + * It may be > 1 due to batches. -- rgerhards, 2009-05-12 */ -rsRetVal qqueueChkPersist(qqueue_t *pThis) +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates) { DEFiRet; - ISOBJ_TYPE_assert(pThis, qqueue); + assert(nUpdates >= 0); + + if(nUpdates == 0) + FINALIZE; - if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { + pThis->iUpdsSincePersist += nUpdates; + if(pThis->iPersistUpdCnt && pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { qqueuePersist(pThis, QUEUE_CHECKPOINT); pThis->iUpdsSincePersist = 0; } +finalize_it: + RETiRet; +} + + +/* persist a queue with all data elements to disk - this is used to handle + * bSaveOnShutdown. We utilize the DA worker to do this. This must only + * be called after all workers have been shut down and if bSaveOnShutdown + * is actually set. Note that this function may potentially run long, + * depending on the queue configuration (e.g. store on remote machine). + * rgerhards, 2009-05-26 + */ +static inline rsRetVal +DoSaveOnShutdown(qqueue_t *pThis) +{ + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + /* we reduce the low water mark, otherwise the DA worker would terminate when + * it is reached. + */ + DBGOPRINT((obj_t*) pThis, "bSaveOnShutdown set, restarting DA worker...\n"); + pThis->bShutdownImmediate = 0; /* would termiante the DA worker! */ + pThis->iLowWtrMrk = 0; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN); /* shutdown worker (only) when done (was _IMMEDIATE!) */ + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* restart DA worker */ + + DBGOPRINT((obj_t*) pThis, "waiting for DA worker to terminate...\n"); + timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); + /* and run the primary queue's DA worker to drain the queue */ + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); + DBGOPRINT((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n", + iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " + "continuing, but results are unpredictable\n", iRetLocal); + } + RETiRet; } @@ -1997,16 +2001,17 @@ rsRetVal qqueueChkPersist(qqueue_t *pThis) /* destructor for the queue object */ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) - pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ - - /* shut down all workers (handles *all* of the persistence logic) - * See function head comment of queueShutdownWorkers () on why we don't call it - * We also do not need to shutdown workers when we are in enqueue-only mode or we are a + /* shut down all workers + * We do not need to shutdown workers when we are in enqueue-only mode or we are a * direct queue - because in both cases we have none... ;) * with a child! -- rgerhards, 2008-01-28 */ if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) - qqueueShutdownWorkers(pThis); + ShutdownWorkers(pThis); + + if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + CHKiRet(DoSaveOnShutdown(pThis)); + } /* finally destruct our (regular) worker thread pool * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, @@ -2042,7 +2047,7 @@ CODESTARTobjDestruct(qqueue) * if need arises (what I doubt...) -- rgerhards, 2008-01-25 */ CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { - dbgoprint((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); + DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); } /* finally, clean up some simple things... */ @@ -2061,11 +2066,8 @@ CODESTARTobjDestruct(qqueue) /* type-specific destructor */ iRet = pThis->qDestruct(pThis); - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); - - if(pThis->pszSpoolDir != NULL) - free(pThis->pszSpoolDir); + free(pThis->pszFilePrefix); + free(pThis->pszSpoolDir); ENDobjDestruct(qqueue) @@ -2079,13 +2081,13 @@ qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) { DEFiRet; - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); + free(pThis->pszFilePrefix); + pThis->pszFilePrefix = NULL; if(pszPrefix == NULL) /* just unset the prefix! */ ABORT_FINALIZE(RS_RET_OK); - if((pThis->pszFilePrefix = malloc(sizeof(uchar) * iLenPrefix + 1)) == NULL) + if((pThis->pszFilePrefix = MALLOC(sizeof(uchar) * iLenPrefix + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFilePrefix, pszPrefix, iLenPrefix + 1); pThis->lenFilePrefix = iLenPrefix; @@ -2115,42 +2117,20 @@ finalize_it: } -/* enqueue a new user data element - * Enqueues the new element and awakes worker thread. +/* enqueue a single data object. + * Note that the queue mutex MUST already be locked when this function is called. + * rgerhards, 2009-06-16 */ -rsRetVal -qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) +static inline rsRetVal +doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) { DEFiRet; - int iCancelStateSave; struct timespec t; - ISOBJ_TYPE_assert(pThis, qqueue); - /* first check if we need to discard this message (which will cause CHKiRet() to exit) - * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize - * and bRunsDA parameters may not reflect the correct settings here, but they are - * "good enough" in the sense that they can be used to drive the decision. Valgrind's - * threading tools may point this access to be an error, but this is done - * intentional. I do not see this causes problems to us. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); - - /* Please note that this function is not cancel-safe and consequently - * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE - * during its execution. If that is not done, race conditions occur if the - * thread is canceled (most important use case is input module termination). - * rgerhards, 2008-01-08 */ - if(pThis->qType != QUEUETYPE_DIRECT) { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(pThis->mut); - } + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); - /* then check if we need to add an assistance disk queue */ - if(pThis->bIsDA) - CHKiRet(qqueueChkStrtDA(pThis)); - /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. * Basic flow control has always been implemented and protects the queue structures @@ -2173,12 +2153,12 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) */ if(flowCtlType == eFLOWCTL_FULL_DELAY) { while(pThis->iQueueSize >= pThis->iFullDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */ } } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) { if(pThis->iQueueSize >= pThis->iLightDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */ } @@ -2192,10 +2172,11 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize) || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0 && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { - dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); timeoutComp(&t, pThis->toEnq); +// TODO : handle enqOnly => discard! if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { - dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); objDestruct(pUsr); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } @@ -2203,7 +2184,39 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis); + +finalize_it: + RETiRet; +} + +/* enqueue multiple user data elements at once. The aim is to provide a faster interface + * for object submission. Uses the multi_submit_t helper object. + * Please note that this function is not cancel-safe and consequently + * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE + * during its execution. If that is not done, race conditions occur if the + * thread is canceled (most important use case is input module termination). + * rgerhards, 2009-06-16 + */ +rsRetVal +qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub) +{ + int iCancelStateSave; + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pMultiSub != NULL); + + if(pThis->qType != QUEUETYPE_DIRECT) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); + } + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { + CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i])); + } + + qqueueChkPersist(pThis, pMultiSub->nElem); finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { @@ -2212,73 +2225,49 @@ finalize_it: /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); - dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); - /* the following pthread_yield is experimental, but brought us performance - * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 - * rgerhards, 2008-10-09 - * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 - */ - if(pThis->bOptimizeUniProc) - pthread_yield(); + DBGOPRINT((obj_t*) pThis, "MultiEnqObj advised worker start\n"); } RETiRet; } -/* set queue mode to enqueue only or not - * There is one subtle issue: this method may be called during queue - * construction or while it is running. In the former case, the queue - * mutex does not yet exist (it is NULL), while in the later case it - * must be locked. The function detects the state and operates as - * required. - * rgerhards, 2008-01-16 +/* enqueue a new user data element + * Enqueues the new element and awakes worker thread. */ -static rsRetVal -qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +rsRetVal +qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) { DEFiRet; - DEFVARS_mutexProtection; + int iCancelStateSave; ISOBJ_TYPE_assert(pThis, qqueue); - /* for simplicity, we do one big mutex lock. This method is extremely seldom - * called, so that doesn't matter... -- rgerhards, 2008-01-16 - */ - if(pThis->mut != NULL) { - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); + if(pThis->qType != QUEUETYPE_DIRECT) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); } - if(bEnqOnly == pThis->bEnqOnly) - FINALIZE; /* no change, nothing to do */ - - if(pThis->bQueueStarted) { - /* we need to adjust queue operation only if we are not during initial param setup */ - if(bEnqOnly == 1) { - /* switch to enqueue-only mode */ - /* this means we need to terminate all workers - that's it... */ - dbgoprint((obj_t*) pThis, "switching to enqueue-only mode, terminating all worker threads\n"); - if(pThis->pWtpReg != NULL) - wtpWakeupAllWrkr(pThis->pWtpReg); - if(pThis->pWtpDA != NULL) - wtpWakeupAllWrkr(pThis->pWtpDA); - } else { - /* switch back to regular mode */ - ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); /* we don't need this so far... */ - } - } + CHKiRet(doEnqSingleObj(pThis, flowCtlType, pUsr)); - pThis->bEnqOnly = bEnqOnly; + qqueueChkPersist(pThis, 1); finalize_it: - if(pThis->mut != NULL) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + if(pThis->qType != QUEUETYPE_DIRECT) { + /* make sure at least one worker is running. */ + qqueueAdviseMaxWorkers(pThis); + /* and release the mutex */ + d_pthread_mutex_unlock(pThis->mut); + pthread_setcancelstate(iCancelStateSave, NULL); + DBGOPRINT((obj_t*) pThis, "EnqueueMsg advised worker start\n"); } + RETiRet; } /* some simple object access methods */ +DEFpropSetMeth(qqueue, bSyncQueueFiles, int) DEFpropSetMeth(qqueue, iPersistUpdCnt, int) DEFpropSetMeth(qqueue, iDeqtWinFromHr, int) DEFpropSetMeth(qqueue, iDeqtWinToHr, int) @@ -2296,6 +2285,7 @@ DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int) DEFpropSetMeth(qqueue, bSaveOnShutdown, int) DEFpropSetMeth(qqueue, pUsr, void*) DEFpropSetMeth(qqueue, iDeqSlowdown, int) +DEFpropSetMeth(qqueue, iDeqBatchSize, int) DEFpropSetMeth(qqueue, sizeOnDiskMax, int64) @@ -2314,8 +2304,6 @@ static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp) if(isProp("iQueueSize")) { pThis->iQueueSize = pProp->val.num; - } else if(isProp("iUngottenObjs")) { - pThis->iUngottenObjs = pProp->val.num; } else if(isProp("tVars.disk.sizeOnDisk")) { pThis->tVars.disk.sizeOnDisk = pProp->val.num; } else if(isProp("tVars.disk.bytesRead")) { @@ -2340,6 +2328,9 @@ rsRetVal qqueueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/queue.h b/runtime/queue.h index a267862d..38c0d491 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -27,8 +27,18 @@ #include <pthread.h> #include "obj.h" #include "wtp.h" +#include "batch.h" #include "stream.h" +/* support for the toDelete list */ +typedef struct toDeleteLst_s toDeleteLst_t; +struct toDeleteLst_s { + qDeqID deqID; + int nElemDeq; /* numbe of elements that were dequeued and as such must now be discarded */ + struct toDeleteLst_s *pNext; +}; + + /* queue types */ typedef enum { QUEUETYPE_FIXED_ARRAY = 0,/* a simple queue made out of a fixed (initially malloced) array fast but memoryhog */ @@ -44,25 +54,15 @@ typedef struct qLinkedList_S { } qLinkedList_t; -typedef struct qWrkThrd_s { - pthread_t thrdID; /* thread ID */ - qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - obj_t *pUsr; /* current user object being processed (or NULL if none) */ - struct queue_s *pQueue; /* my queue (important if only the work thread instance is passed! */ - int iThrd; /* my worker thread array index */ - pthread_cond_t condInitDone; /* signaled when the thread startup is done (once per thread existance) */ - pthread_mutex_t mut; -} qWrkThrd_t; /* type for queue worker threads */ - /* the queue object */ -typedef struct queue_s { +struct queue_s { BEGINobjInstance; queueType_t qType; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ - int bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ - int bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ - int bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ - int bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ + int nLogDeq; /* number of elements currently logically dequeued */ + int bShutdownImmediate; /* should all workers cease processing messages? */ + sbool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ + sbool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ + sbool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ int iQueueSize; /* Current number of elements in the queue */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ @@ -73,17 +73,20 @@ typedef struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ + sbool bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ int iFullDlyMrk; /* if the queue is above this mark, FULL_DELAYable message are put on hold */ int iLightDlyMrk; /* if the queue is above this mark, LIGHT_DELAYable message are put on hold */ int iDiscardSeverity;/* messages of this severity above are discarded on too-full queue */ - int bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ + sbool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ int toQShutdown; /* timeout for regular queue shutdown in ms */ int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ + toDeleteLst_t *toDeleteLst;/* this queue's to-delete list */ int toEnq; /* enqueue timeout */ + int iDeqBatchSize; /* max number of elements that shall be dequeued at once */ /* rate limiting settings (will be expanded) */ int iDeqSlowdown; /* slow down dequeue by specified nbr of microseconds */ /* end rate limiting */ @@ -97,18 +100,19 @@ typedef struct queue_s { * applied to detect user configuration errors (and tell me how should we detect what * the user really wanted...). -- rgerhards, 2008-04-02 */ - /* ane dequeue time window */ - rsRetVal (*pConsumer)(void *,void*); /* user-supplied consumer function for dequeued messages */ + /* end dequeue time window */ + rsRetVal (*pConsumer)(void *,batch_t*,int*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the - * user pointer that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 is pointer - * to message) - * rgerhards, 2008-01-28 + * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 + * is pointer to an array of message message pointers), arg3 is a pointer to an interger which is zero + * during normal operations and one if the consumer must urgently shut down. */ /* type-specific handlers (set during construction) */ rsRetVal (*qConstruct)(struct queue_s *pThis); rsRetVal (*qDestruct)(struct queue_s *pThis); rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr); - rsRetVal (*qDel)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDel)(struct queue_s *pThis); /* end type-specific handler */ /* synchronization variables */ pthread_mutex_t mutThrdMgmt; /* mutex for the queue's thread management */ @@ -117,7 +121,6 @@ typedef struct queue_s { pthread_cond_t belowFullDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t belowLightDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t condDAReady;/* signalled when the DA queue is fully initialized and ready for processing */ - int bChildIsDone; /* set to 1 when the child DA queue has finished processing, 0 otherwise */ int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* the following variables are always present, because they @@ -132,42 +135,33 @@ typedef struct queue_s { int iNumberFiles; /* how many files make up the queue? */ int64 iMaxFileSize; /* max size for a single queue file */ int64 sizeOnDiskMax; /* maximum size on disk allowed */ + qDeqID deqIDAdd; /* next dequeue ID to use during add to queue store */ + qDeqID deqIDDel; /* queue store delete position */ int bIsDA; /* is this queue disk assisted? */ - int bRunsDA; /* is this queue actually *running* disk assisted? */ struct queue_s *pqDA; /* queue for disk-assisted modes */ struct queue_s *pqParent;/* pointer to the parent (if this is a child queue) */ int bDAEnqOnly; /* EnqOnly setting for DA queue */ - /* some data elements for the queueUngetObj() functionality. This list should always be short - * and is always kept in memory - */ - qLinkedList_t *pUngetRoot; - qLinkedList_t *pUngetLast; - int iUngottenObjs; /* number of objects currently in the "ungotten" list */ /* now follow queueing mode specific data elements */ union { /* different data elements based on queue type (qType) */ struct { - long head, tail; + long deqhead, head, tail; void** pBuf; /* the queued user data structure */ } farray; struct { - qLinkedList_t *pRoot; + qLinkedList_t *pDeqRoot; + qLinkedList_t *pDelRoot; qLinkedList_t *pLast; } linklist; struct { int64 sizeOnDisk; /* current amount of disk space used */ int64 bytesRead; /* number of bytes read from current (undeleted!) file */ - strm_t *pWrite; /* current file to be written */ - strm_t *pRead; /* current file to be read */ + strm_t *pWrite; /* current file to be written */ + strm_t *pReadDeq; /* current file for dequeueing */ + strm_t *pReadDel; /* current file for deleting */ } disk; } tVars; -} qqueue_t; - -/* some symbolic constants for easier reference */ -#define QUEUE_MODE_ENQDEQ 0 -#define QUEUE_MODE_ENQONLY 1 +}; -#define QUEUE_IDX_DA_WORKER 0 /* index for the DA worker (fixed) */ -#define QUEUE_PTR_DA_WORKER(x) (&((pThis)->pWrkThrds[0])) /* the define below is an "eternal" timeout for the timeout settings which require a value. * It is one day, which is not really eternal, but comes close to it if we think about @@ -178,14 +172,16 @@ typedef struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); +rsRetVal qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub); rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*)); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); +PROTOTYPEpropSetMeth(qqueue, bSyncQueueFiles, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinToHr, int); PROTOTYPEpropSetMeth(qqueue, toQShutdown, long); @@ -201,6 +197,7 @@ PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int); PROTOTYPEpropSetMeth(qqueue, pUsr, void*); PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int); PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); +PROTOTYPEpropSetMeth(qqueue, iDeqBatchSize, int); #define qqueueGetID(pThis) ((unsigned long) pThis) #endif /* #ifndef QUEUE_H_INCLUDED */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 8df100a1..a76ae25a 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -77,6 +77,10 @@ #include "conf.h" #include "glbl.h" #include "errmsg.h" +#include "prop.h" +#include "rule.h" +#include "ruleset.h" +#include "parser.h" /* forward definitions */ static rsRetVal dfltErrLogger(int, uchar *errMsg); @@ -144,20 +148,16 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) * class immediately after it is initialized. And, of course, we load those classes * first that we use ourselfs... -- rgerhards, 2008-03-07 */ + if(ppErrObj != NULL) *ppErrObj = "prop"; + CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; CHKiRet(glblClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "datetime"; - CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "str,"; - CHKiRet(strmClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wti"; - CHKiRet(wtiClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wtp"; - CHKiRet(wtpClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "queue"; - CHKiRet(qqueueClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok_token"; + CHKiRet(ctok_tokenClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok"; + CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmstk"; CHKiRet(vmstkClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "sysvar"; @@ -168,14 +168,22 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(vmopClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmprg"; CHKiRet(vmprgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok_token"; - CHKiRet(ctok_tokenClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok"; - CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "expr"; CHKiRet(exprClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "rule"; + CHKiRet(ruleClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ruleset"; + CHKiRet(rulesetClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wti"; + CHKiRet(wtiClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wtp"; + CHKiRet(wtpClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "queue"; + CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "parser"; + CHKiRet(parserClassInit(NULL)); /* dummy "classes" */ if(ppErrObj != NULL) *ppErrObj = "str"; @@ -206,6 +214,8 @@ rsrtExit(void) /* do actual de-init only if we are the last runtime user */ confClassExit(); glblClassExit(); + rulesetClassExit(); + ruleClassExit(); objClassExit(); /* *THIS* *MUST/SHOULD?* always be the first class initilizer being called (except debug)! */ } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 77d845fd..a75d2bc0 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -29,7 +29,29 @@ /* ############################################################# * * # Config Settings # * * ############################################################# */ -#define RS_STRINGBUF_ALLOC_INCREMENT 128 +#define RS_STRINGBUF_ALLOC_INCREMENT 128 +/* MAXSIZE are absolute maxima, while BUFSIZE are just values after which + * processing is more time-intense. The BUFSIZE params currently add their + * value to the fixed size of the message object. + */ +#define CONF_TAG_MAXSIZE 512 /* a value that is deemed far too large for any valid TAG */ +#define CONF_HOSTNAME_MAXSIZE 512 /* a value that is deemed far too large for any valid HOSTNAME */ +#define CONF_RAWMSG_BUFSIZE 101 +#define CONF_TAG_BUFSIZE 32 +#define CONF_HOSTNAME_BUFSIZE 32 +#define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ +#define CONF_MIN_SIZE_FOR_COMPRESS 60 /* config param: minimum message size to try compression. The smaller + * the message, the less likely is any compression gain. We check for + * gain before we submit the message. But to do so we still need to + * do the (costly) compress() call. The following setting sets a size + * for which no call to compress() is done at all. This may result in + * a few more bytes being transmited but better overall performance. + * Note: I have not yet checked the minimum UDP packet size. It might be + * that we do not save anything by compressing very small messages, because + * UDP might need to pad ;) + * rgerhards, 2006-11-30 + */ + /* ############################################################# * * # End Config Settings # * @@ -58,15 +80,34 @@ #endif +/* the rsyslog core provides information about present feature to plugins + * asking it. Below are feature-test macros which must be used to query + * features. Note that this must be powers of two, so that multiple queries + * can be combined. -- rgerhards, 2009-04-27 + */ +#define CORE_FEATURE_BATCHING 1 +/*#define CORE_FEATURE_whatever 2 ... and so on ... */ + +/* some universal fixed size integer defines ... */ +typedef long long int64; +typedef long long unsigned uint64; +typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef char intTiny; /* 0..127! */ +typedef unsigned char uintTiny; /* 0..255! */ + /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ +typedef struct aUsrp_s aUsrp_t; typedef struct thrdInfo thrdInfo_t; typedef struct obj_s obj_t; -typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ +typedef struct ruleset_s ruleset_t; +typedef struct rule_s rule_t; +//typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ typedef struct NetAddr netAddr_t; typedef struct netstrms_s netstrms_t; typedef struct netstrm_s netstrm_t; typedef struct nssel_s nssel_t; +typedef struct nspoll_s nspoll_t; typedef enum nsdsel_waitOp_e nsdsel_waitOp_t; typedef struct nsd_ptcp_s nsd_ptcp_t; typedef struct nsd_gtls_s nsd_gtls_t; @@ -74,9 +115,14 @@ typedef struct nsd_gsspi_s nsd_gsspi_t; typedef struct nsd_nss_s nsd_nss_t; typedef struct nsdsel_ptcp_s nsdsel_ptcp_t; typedef struct nsdsel_gtls_s nsdsel_gtls_t; +typedef struct nsdpoll_ptcp_s nsdpoll_ptcp_t; +typedef struct wti_s wti_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; +typedef obj_t nsdpoll_t; typedef struct msg msg_t; +typedef struct queue_s qqueue_t; +typedef struct prop_s prop_t; typedef struct interface_s interface_t; typedef struct objInfo_s objInfo_t; typedef enum rsRetVal_ rsRetVal; /**< friendly type for global return value */ @@ -85,21 +131,30 @@ typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; typedef struct tcps_sess_s tcps_sess_t; +typedef struct strmsrv_s strmsrv_t; +typedef struct strms_sess_s strms_sess_t; typedef struct vmstk_s vmstk_t; +typedef struct batch_obj_s batch_obj_t; +typedef struct batch_s batch_t; +typedef struct wtp_s wtp_t; +typedef struct modInfo_s modInfo_t; +typedef struct parser_s parser_t; +typedef struct parserList_s parserList_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ +typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? - -/* some universal 64 bit define... */ -typedef long long int64; -typedef long long unsigned uint64; -typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif +typedef struct epoll_event epoll_event_t; + +typedef char sbool; /* (small bool) I intentionally use char, to keep it slim so that many fit into the CPU cache! */ + /* settings for flow control * TODO: is there a better place for them? -- rgerhards, 2008-03-14 */ @@ -109,6 +164,72 @@ typedef enum { eFLOWCTL_FULL_DELAY = 2 /**< delay possible for extended period of time */ } flowControl_t; +/* filter operations */ +typedef enum { + FIOP_NOP = 0, /* do not use - No Operation */ + FIOP_CONTAINS = 1, /* contains string? */ + FIOP_ISEQUAL = 2, /* is (exactly) equal? */ + FIOP_STARTSWITH = 3, /* starts with a string? */ + FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ + FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ +} fiop_t; + + +/* multi-submit support. + * This is done via a simple data structure, which holds the number of elements + * as well as an array of to-be-submitted messages. + * rgerhards, 2009-06-16 + */ +typedef struct multi_submit_s multi_submit_t; +struct multi_submit_s { + short maxElem; /* maximum number of Elements */ + short nElem; /* current number of Elements, points to the next one FREE */ + msg_t **ppMsgs; +}; + + +#ifndef _PATH_CONSOLE +#define _PATH_CONSOLE "/dev/console" +#endif + +/* properties are now encoded as (tiny) integers. I do not use an enum as I would like + * to keep the memory footprint small (and thus cache hits high). + * rgerhards, 2009-06-26 + */ +typedef uintTiny propid_t; +#define PROP_INVALID 0 +#define PROP_MSG 1 +#define PROP_TIMESTAMP 2 +#define PROP_HOSTNAME 3 +#define PROP_SYSLOGTAG 4 +#define PROP_RAWMSG 5 +#define PROP_INPUTNAME 6 +#define PROP_FROMHOST 7 +#define PROP_FROMHOST_IP 8 +#define PROP_PRI 9 +#define PROP_PRI_TEXT 10 +#define PROP_IUT 11 +#define PROP_SYSLOGFACILITY 12 +#define PROP_SYSLOGFACILITY_TEXT 13 +#define PROP_SYSLOGSEVERITY 14 +#define PROP_SYSLOGSEVERITY_TEXT 15 +#define PROP_TIMEGENERATED 16 +#define PROP_PROGRAMNAME 17 +#define PROP_PROTOCOL_VERSION 18 +#define PROP_STRUCTURED_DATA 19 +#define PROP_APP_NAME 20 +#define PROP_PROCID 21 +#define PROP_MSGID 22 +#define PROP_SYS_NOW 150 +#define PROP_SYS_YEAR 151 +#define PROP_SYS_MONTH 152 +#define PROP_SYS_DAY 153 +#define PROP_SYS_HOUR 154 +#define PROP_SYS_HHOUR 155 +#define PROP_SYS_QHOUR 156 +#define PROP_SYS_MINUTE 157 +#define PROP_SYS_MYHOSTNAME 158 + /* The error codes below are orginally "borrowed" from * liblogging. As such, we reserve values up to -2999 @@ -270,16 +391,45 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ + RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ + RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ + RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */ + RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */ + RS_RET_NON_SIZELIMITCMD = -2125, /**< size limit for file defined, but no size limit command given */ + RS_RET_SIZELIMITCMD_DIDNT_RESOLVE = -2126, /**< size limit command did not resolve situation */ + RS_RET_STREAM_DISABLED = -2127, /**< a file has been disabled (e.g. by size limit restriction) */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ + RS_RET_ZLIB_ERR = -2141, /**< error during zlib call */ + RS_RET_VAR_NOT_FOUND = -2142, /**< variable not found */ + RS_RET_EMPTY_MSG = -2143, /**< provided (raw) MSG is empty */ + RS_RET_PEER_CLOSED_CONN = -2144, /**< remote peer closed connection (information, no error) */ + RS_RET_NO_SRCNAME_TPL = -2150, /**< sourcename template was not specified where one was needed (omudpspoof spoof addr) */ + RS_RET_HOST_NOT_SPECIFIED = -2151, /**< (target) host was not specified where it was needed */ + RS_RET_ERR_LIBNET_INIT = -2152, /**< error initializing libnet */ + RS_RET_FORCE_TERM = -2153, /**< thread was forced to terminate by bShallShutdown, a state, not an error */ + RS_RET_RULES_QUEUE_EXISTS = -2154,/**< we were instructed to create a new ruleset queue, but one already exists */ + RS_RET_NO_CURR_RULESET = -2155,/**< no current ruleset exists (but one is required) */ + RS_RET_NO_MSG_PASSING = -2156,/**< output module interface parameter passing mode "MSG" is not available but required */ + RS_RET_RULESET_NOT_FOUND = -2157,/**< a required ruleset could not be found */ + RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */ + RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */ + RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */ + RS_RET_EINTR = -2161, /**< EINTR occured during a system call (not necessarily an error) */ + RS_RET_ERR_EPOLL = -2162, /**< epoll() returned with an unexpected error code */ + RS_RET_ERR_EPOLL_CTL = -2163, /**< epol_ctll() returned with an unexpected error code */ + RS_RET_TIMEOUT = -2164, /**< timeout occured during operation */ + RS_RET_RCV_ERR = -2165, /**< error occured during socket rcv operation */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ /* some generic error/status codes */ + RS_RET_OK = 0, /**< operation successful */ RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */ RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */ RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */ - RS_RET_OK = 0 /**< operation successful */ + RS_RET_IDLE = 4, /**< operation successful, but callee is idle (e.g. because queue is empty) */ + RS_RET_TERMINATE_WHEN_IDLE = 5 /**< operation successful, function is requested to terminate when idle */ }; /* some helpful macros to work with srRetVals. @@ -359,6 +509,10 @@ typedef enum rsObjectID rsObjID; # define O_CLOEXEC 0 #endif +/* some constants */ +#define MUTEX_ALREADY_LOCKED 0 +#define LOCK_MUTEX 1 + /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); diff --git a/runtime/rule.c b/runtime/rule.c new file mode 100644 index 00000000..65ad071e --- /dev/null +++ b/runtime/rule.c @@ -0,0 +1,479 @@ +/* rule.c - rsyslog's rule object + * + * See file comment in rule.c for the overall structure of rule processing. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <ctype.h> + +#include "rsyslog.h" +#include "obj.h" +#include "action.h" +#include "rule.h" +#include "errmsg.h" +#include "vm.h" +#include "var.h" +#include "srUtils.h" +#include "unicode-helper.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(expr) +DEFobjCurrIf(var) +DEFobjCurrIf(vm) + + +/* support for simple textual representation of FIOP names + * rgerhards, 2005-09-27 + */ +static char* +getFIOPName(unsigned iFIOP) +{ + char *pRet; + switch(iFIOP) { + case FIOP_CONTAINS: + pRet = "contains"; + break; + case FIOP_ISEQUAL: + pRet = "isequal"; + break; + case FIOP_STARTSWITH: + pRet = "startswith"; + break; + case FIOP_REGEX: + pRet = "regex"; + break; + default: + pRet = "NOP"; + break; + } + return pRet; +} + + +/* iterate over all actions, this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + return llExecFunc(&pThis->llActList, pFunc, pParam); +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +typedef struct processMsgDoActions_s { + int bPrevWasSuspended; /* was the previous action suspended? */ + msg_t *pMsg; +} processMsgDoActions_t; +DEFFUNC_llExecFunc(processMsgDoActions) +{ + DEFiRet; + rsRetVal iRetMod; /* return value of module - we do not always pass that back */ + action_t *pAction = (action_t*) pData; + processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; + + assert(pAction != NULL); + + if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { + dbgprintf("not calling action because the previous one is not suspended\n"); + ABORT_FINALIZE(RS_RET_OK); + } + + iRetMod = actionCallAction(pAction, pDoActData->pMsg); + if(iRetMod == RS_RET_DISCARDMSG) { + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } else if(iRetMod == RS_RET_SUSPENDED) { + /* indicate suspension for next module to be called */ + pDoActData->bPrevWasSuspended = 1; + } else { + pDoActData->bPrevWasSuspended = 0; + } + +finalize_it: + RETiRet; +} + + +/* This functions looks at the given message and checks if it matches the + * provided filter condition. + */ +static rsRetVal +shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) +{ + DEFiRet; + unsigned short pbMustBeFreed; + uchar *pszPropVal; + int bRet = 0; + size_t propLen; + vm_t *pVM = NULL; + var_t *pResult = NULL; + + ISOBJ_TYPE_assert(pRule, rule); + assert(pMsg != NULL); + + /* we first have a look at the global, BSD-style block filters (for tag + * and host). Only if they match, we evaluate the actual filter. + * rgerhards, 2005-10-18 + */ + if(pRule->eHostnameCmpMode == HN_NO_COMP) { + /* EMPTY BY INTENSION - we check this value first, because + * it is the one most often used, so this saves us time! + */ + } else if(pRule->eHostnameCmpMode == HN_COMP_MATCH) { + if(rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '+%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } else { /* must be -hostname */ + if(!rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '-%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } + +RUNLOG_VAR("%p", pRule->pCSProgNameComp); + if(pRule->pCSProgNameComp != NULL) { + int bInv = 0, bEqv = 0, offset = 0; + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp)) == '-') { + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp) + 1) == '-') + offset = 1; + else { + bInv = 1; + offset = 1; + } + } + if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, + (uchar*) getProgramName(pMsg, LOCK_MUTEX), getProgramNameLen(pMsg, LOCK_MUTEX))) + bEqv = 1; + + if((!bEqv && !bInv) || (bEqv && bInv)) { + /* not equal or inverted selection, so we are already done... */ + DBGPRINTF("programname filter '%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg, LOCK_MUTEX)); + FINALIZE; + } + } + + /* done with the BSD-style block filters */ + + if(pRule->f_filter_type == FILTER_PRI) { + /* skip messages that are incorrect priority */ +dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility]); + if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ + ((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) ) + bRet = 0; + else + bRet = 1; + } else if(pRule->f_filter_type == FILTER_EXPR) { + CHKiRet(vm.Construct(&pVM)); + CHKiRet(vm.ConstructFinalize(pVM)); + CHKiRet(vm.SetMsg(pVM, pMsg)); + CHKiRet(vm.ExecProg(pVM, pRule->f_filterData.f_expr->pVmprg)); + CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); + dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + /* VM is destructed on function exit */ + bRet = (pResult->val.num) ? 1 : 0; + } else { + assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ + pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.propID, &propLen, &pbMustBeFreed); + + /* Now do the compares (short list currently ;)) */ + switch(pRule->f_filterData.prop.operation ) { + case FIOP_CONTAINS: + if(rsCStrLocateInSzStr(pRule->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) + bRet = 1; + break; + case FIOP_ISEQUAL: + if(rsCStrSzStrCmp(pRule->f_filterData.prop.pCSCompValue, + pszPropVal, ustrlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_STARTSWITH: + if(rsCStrSzStrStartsWithCStr(pRule->f_filterData.prop.pCSCompValue, + pszPropVal, ustrlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_REGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 0, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 1, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + default: + /* here, it handles NOP (for performance reasons) */ + assert(pRule->f_filterData.prop.operation == FIOP_NOP); + bRet = 1; /* as good as any other default ;) */ + break; + } + + /* now check if the value must be negated */ + if(pRule->f_filterData.prop.isNegated) + bRet = (bRet == 1) ? 0 : 1; + + if(Debug) { + dbgprintf("Filter: check for property '%s' (value '%s') ", + propIDToName(pRule->f_filterData.prop.propID), pszPropVal); + if(pRule->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("%s '%s': %s\n", + getFIOPName(pRule->f_filterData.prop.operation), + rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue), + bRet ? "TRUE" : "FALSE"); + } + + /* cleanup */ + if(pbMustBeFreed) + free(pszPropVal); + } + +finalize_it: + /* destruct in any case, not just on error, but it makes error handling much easier */ + if(pVM != NULL) + vm.Destruct(&pVM); + + if(pResult != NULL) + var.Destruct(&pResult); + + *bProcessMsg = bRet; + RETiRet; +} + + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(rule_t *pThis, msg_t *pMsg) +{ + int bProcessMsg; + processMsgDoActions_t DoActData; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, rule); + assert(pMsg != NULL); + + /* first check the filters... */ + CHKiRet(shouldProcessThisMessage(pThis, pMsg, &bProcessMsg)); + if(bProcessMsg) { + DoActData.pMsg = pMsg; + DoActData.bPrevWasSuspended = 0; + CHKiRet(llExecFunc(&pThis->llActList, processMsgDoActions, (void*)&DoActData)); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(rule) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(rule) + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +ruleConstructFinalize(rule_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + + /* note: actionDestruct is from action.c API! */ + CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +/* destructor for the rule object */ +BEGINobjDestruct(rule) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(rule) + if(pThis->pCSHostnameComp != NULL) + rsCStrDestruct(&pThis->pCSHostnameComp); + if(pThis->pCSProgNameComp != NULL) + rsCStrDestruct(&pThis->pCSProgNameComp); + + if(pThis->f_filter_type == FILTER_PROP) { + if(pThis->f_filterData.prop.pCSCompValue != NULL) + rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); + if(pThis->f_filterData.prop.regex_cache != NULL) + rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); + } else if(pThis->f_filter_type == FILTER_EXPR) { + if(pThis->f_filterData.f_expr != NULL) + expr.Destruct(&pThis->f_filterData.f_expr); + } + + llDestroy(&pThis->llActList); +ENDobjDestruct(rule) + + +/* set the associated ruleset */ +static rsRetVal +setAssRuleset(rule_t *pThis, ruleset_t *pRuleset) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + ISOBJ_TYPE_assert(pRuleset, ruleset); + pThis->pRuleset = pRuleset; + RETiRet; +} + +/* get the associated ruleset (may be NULL if not set!) */ +static ruleset_t* +getAssRuleset(rule_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, rule); + return pThis->pRuleset; +} + + +/* helper to DebugPrint, to print out all actions via + * the llExecFunc() facility. + */ +DEFFUNC_llExecFunc(dbgPrintInitInfoAction) +{ + DEFiRet; + iRet = actionDbgPrint((action_t*) pData); + dbgprintf("\n"); + RETiRet; +} + + +/* debugprint for the rule object */ +BEGINobjDebugPrint(rule) /* be sure to specify the object type also in END and CODESTART macros! */ + int i; +CODESTARTobjDebugPrint(rule) + dbgoprint((obj_t*) pThis, "rsyslog rule:\n"); + if(pThis->pCSProgNameComp != NULL) + dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(pThis->pCSProgNameComp)); + if(pThis->eHostnameCmpMode != HN_NO_COMP) + dbgprintf("hostname: %s '%s'\n", + pThis->eHostnameCmpMode == HN_COMP_MATCH ? + "only" : "allbut", + rsCStrGetSzStrNoNULL(pThis->pCSHostnameComp)); + if(pThis->f_filter_type == FILTER_PRI) { + for (i = 0; i <= LOG_NFACILITIES; i++) + if (pThis->f_filterData.f_pmask[i] == TABLE_NOPRI) + dbgprintf(" X "); + else + dbgprintf("%2X ", pThis->f_filterData.f_pmask[i]); + } else if(pThis->f_filter_type == FILTER_EXPR) { + dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); + } else { + dbgprintf("PROPERTY-BASED Filter:\n"); + dbgprintf("\tProperty.: '%s'\n", propIDToName(pThis->f_filterData.prop.propID)); + dbgprintf("\tOperation: "); + if(pThis->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("'%s'\n", getFIOPName(pThis->f_filterData.prop.operation)); + dbgprintf("\tValue....: '%s'\n", + rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue)); + dbgprintf("\tAction...: "); + } + + dbgprintf("\nActions:\n"); + llExecFunc(&pThis->llActList, dbgPrintInitInfoAction, NULL); /* actions */ + + dbgprintf("\n"); +ENDobjDebugPrint(rule) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(rule) +CODESTARTobjQueryInterface(rule) + if(pIf->ifVersion != ruleCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = ruleConstruct; + pIf->ConstructFinalize = ruleConstructFinalize; + pIf->Destruct = ruleDestruct; + pIf->DebugPrint = ruleDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->ProcessMsg = processMsg; + pIf->SetAssRuleset = setAssRuleset; + pIf->GetAssRuleset = getAssRuleset; +finalize_it: +ENDobjQueryInterface(rule) + + +/* Exit the rule class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(rule, OBJ_IS_CORE_MODULE) /* class, version */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(expr, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vm, CORE_COMPONENT); +ENDObjClassExit(rule) + + +/* Initialize the rule class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(rule, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(expr, CORE_COMPONENT)); + CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, ruleDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, ruleConstructFinalize); +ENDObjClassInit(rule) + +/* vi:set ai: + */ diff --git a/runtime/rule.h b/runtime/rule.h new file mode 100644 index 00000000..7b607637 --- /dev/null +++ b/runtime/rule.h @@ -0,0 +1,77 @@ +/* The rule object. + * + * This implements rules within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULE_H +#define INCLUDED_RULE_H + +#include "linkedlist.h" +#include "regexp.h" +#include "expr.h" + +/* the rule object */ +struct rule_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + /* filter properties */ + enum { + FILTER_PRI = 0, /* traditional PRI based filer */ + FILTER_PROP = 1, /* extended filter, property based */ + FILTER_EXPR = 2 /* extended filter, expression based */ + } f_filter_type; + EHostnameCmpMode eHostnameCmpMode; + cstr_t *pCSHostnameComp; /* hostname to check */ + cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ + union { + u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ + struct { + fiop_t operation; + regex_t *regex_cache; /* cache for compiled REs, if such are used */ + cstr_t *pCSCompValue; /* value to "compare" against */ + sbool isNegated; + propid_t propID; /* ID of the requested property */ + } prop; + expr_t *f_expr; /* expression object */ + } f_filterData; + + ruleset_t *pRuleset; /* associated ruleset */ + linkedList_t llActList; /* list of configured actions */ +}; + +/* interfaces */ +BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(rule); + rsRetVal (*Construct)(rule_t **ppThis); + rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(rule_t **ppThis); + rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); + rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg); + rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*); + ruleset_t* (*GetAssRuleset)(rule_t *pThis); +ENDinterface(rule) +#define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(rule); + +#endif /* #ifndef INCLUDED_RULE_H */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c new file mode 100644 index 00000000..1a77be2b --- /dev/null +++ b/runtime/ruleset.c @@ -0,0 +1,564 @@ +/* ruleset.c - rsyslog's ruleset object + * + * We have a two-way structure of linked lists: one global linked list + * (llAllRulesets) hold alls rule sets that we know. Included in each + * list is a list of rules (which contain a list of actions, but that's + * a different story). + * + * Usually, only a single rule set is executed. However, there exist some + * situations where all rules must be iterated over, for example on HUP. Thus, + * we also provide interfaces to do that. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <ctype.h> + +#include "rsyslog.h" +#include "obj.h" +#include "cfsysline.h" +#include "msg.h" +#include "ruleset.h" +#include "rule.h" +#include "errmsg.h" +#include "parser.h" +#include "unicode-helper.h" +#include "dirty.h" /* for main ruleset queue creation */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(rule) +DEFobjCurrIf(parser) + +linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ +ruleset_t *pCurrRuleset = NULL; /* currently "active" ruleset */ +ruleset_t *pDfltRuleset = NULL; /* current default ruleset, e.g. for binding to actions which have no other */ + +/* ---------- linked-list key handling functions ---------- */ + +/* destructor for linked list keys. + */ +static rsRetVal keyDestruct(void __attribute__((unused)) *pData) +{ + free(pData); + return RS_RET_OK; +} + + +/* ---------- END linked-list key handling functions ---------- */ + + +/* driver to iterate over all of this ruleset actions */ +typedef struct iterateAllActions_s { + rsRetVal (*pFunc)(void*, void*); + void *pParam; +} iterateAllActions_t; +DEFFUNC_llExecFunc(doIterateRulesetActions) +{ + DEFiRet; + rule_t* pRule = (rule_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over all actions of THIS rule set. + */ +static rsRetVal +iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&(pThis->llRules), doIterateRulesetActions, ¶ms)); + +finalize_it: + RETiRet; +} + + +/* driver to iterate over all actions */ +DEFFUNC_llExecFunc(doIterateAllActions) +{ + DEFiRet; + ruleset_t* pThis = (ruleset_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over ALL actions present in the WHOLE system. + * this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + +finalize_it: + RETiRet; +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +DEFFUNC_llExecFunc(processMsgDoRules) +{ + rsRetVal iRet; + ISOBJ_TYPE_assert(pData, rule); + iRet = rule.ProcessMsg((rule_t*) pData, (msg_t*) pParam); +dbgprintf("ruleset: get iRet %d from rule.ProcessMsg()\n", iRet); + return iRet; +} + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(msg_t *pMsg) +{ + ruleset_t *pThis; + DEFiRet; + assert(pMsg != NULL); + + pThis = (pMsg->pRuleset == NULL) ? pDfltRuleset : pMsg->pRuleset; + ISOBJ_TYPE_assert(pThis, ruleset); + + CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); + +finalize_it: +dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet); + RETiRet; +} + + +/* return the ruleset-assigned parser list. NULL means use the default + * parser list. + * rgerhards, 2009-11-04 + */ +static parserList_t* +GetParserList(msg_t *pMsg) +{ + return (pMsg->pRuleset == NULL) ? pDfltRuleset->pParserLst : pMsg->pRuleset->pParserLst; +} + + +/* Add a new rule to the end of the current rule set. We do a number + * of checks and ignore the rule if it does not pass them. + */ +static rsRetVal +addRule(ruleset_t *pThis, rule_t **ppRule) +{ + int iActionCnt; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, ruleset); + ISOBJ_TYPE_assert(*ppRule, rule); + + CHKiRet(llGetNumElts(&(*ppRule)->llActList, &iActionCnt)); + if(iActionCnt == 0) { + errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); + rule.Destruct(ppRule); + } else { + CHKiRet(llAppend(&pThis->llRules, NULL, *ppRule)); + dbgprintf("selector line successfully processed\n"); + } + +finalize_it: + RETiRet; +} + + +/* set name for ruleset */ +static rsRetVal setName(ruleset_t *pThis, uchar *pszName) +{ + DEFiRet; + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); + +finalize_it: + RETiRet; +} + + +/* get current ruleset + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static ruleset_t* +GetCurrent(void) +{ + return pCurrRuleset; +} + + +/* get main queue associated with ruleset. If no ruleset-specifc main queue + * is set, the primary main message queue is returned. + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static qqueue_t* +GetRulesetQueue(ruleset_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, ruleset); + return (pThis->pQueue == NULL) ? pMsgQueue : pThis->pQueue; +} + + +/* Find the ruleset with the given name and return a pointer to its object. + */ +static rsRetVal +GetRuleset(ruleset_t **ppRuleset, uchar *pszName) +{ + DEFiRet; + assert(ppRuleset != NULL); + assert(pszName != NULL); + + CHKiRet(llFind(&llRulesets, pszName, (void*) ppRuleset)); + +finalize_it: + RETiRet; +} + + +/* Set a new default rule set. If the default can not be found, no change happens. + */ +static rsRetVal +SetDefaultRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pDfltRuleset = pRuleset; + dbgprintf("default rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + +/* Set a new current rule set. If the ruleset can not be found, no change happens. + */ +static rsRetVal +SetCurrRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pCurrRuleset = pRuleset; + dbgprintf("current rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + +/* destructor we need to destruct rules inside our linked list contents. + */ +static rsRetVal +doRuleDestruct(void *pData) +{ + rule_t *pRule = (rule_t *) pData; + DEFiRet; + rule.Destruct(&pRule); + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(ruleset) /* be sure to specify the object type also in END macro! */ + CHKiRet(llInit(&pThis->llRules, doRuleDestruct, NULL, NULL)); +finalize_it: +ENDobjConstruct(ruleset) + + +/* ConstructionFinalizer + * This also adds the rule set to the list of all known rulesets. + */ +static rsRetVal +rulesetConstructFinalize(ruleset_t *pThis) +{ + uchar *keyName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + + /* we must duplicate our name, as the key destructer would also + * free it, resulting in a double-free. It's also cleaner to have + * two separate copies. + */ + CHKmalloc(keyName = ustrdup(pThis->pszName)); + CHKiRet(llAppend(&llRulesets, keyName, pThis)); + + /* this now also is the new current ruleset */ + pCurrRuleset = pThis; + + /* and also the default, if so far none has been set */ + if(pDfltRuleset == NULL) + pDfltRuleset = pThis; + +finalize_it: + RETiRet; +} + + +/* destructor for the ruleset object */ +BEGINobjDestruct(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(ruleset) + dbgprintf("destructing ruleset %p, name %p\n", pThis, pThis->pszName); + if(pThis->pQueue != NULL) { + qqueueDestruct(&pThis->pQueue); + } + if(pThis->pParserLst != NULL) { + parser.DestructParserList(&pThis->pParserLst); + } + llDestroy(&pThis->llRules); + free(pThis->pszName); +ENDobjDestruct(ruleset) + +/* this is a special destructor for the linkedList class. LinkedList does NOT + * provide a pointer to the pointer, but rather the raw pointer itself. So we + * must map this, otherwise the destructor will abort. + */ +static rsRetVal +rulesetDestructForLinkedList(void *pData) +{ + ruleset_t *pThis = (ruleset_t*) pData; + return rulesetDestruct(&pThis); +} + + +/* destruct ALL rule sets that reside in the system. This must + * be callable before unloading this module as the module may + * not be unloaded before unload of the actions is required. This is + * kind of a left-over from previous logic and may be optimized one + * everything runs stable again. -- rgerhards, 2009-06-10 + */ +static rsRetVal +destructAllActions(void) +{ + DEFiRet; + + CHKiRet(llDestroy(&llRulesets)); + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + +finalize_it: + RETiRet; +} + +/* helper for debugPrint(), initiates rule printing */ +DEFFUNC_llExecFunc(doDebugPrintRule) +{ + return rule.DebugPrint((rule_t*) pData); +} +/* debugprint for the ruleset object */ +BEGINobjDebugPrint(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(ruleset) + dbgoprint((obj_t*) pThis, "rsyslog ruleset %s:\n", pThis->pszName); + llExecFunc(&pThis->llRules, doDebugPrintRule, NULL); +ENDobjDebugPrint(ruleset) + + +/* helper for debugPrintAll(), prints a single ruleset */ +DEFFUNC_llExecFunc(doDebugPrintAll) +{ + return rulesetDebugPrint((ruleset_t*) pData); +} +/* debug print all rulesets + */ +static rsRetVal +debugPrintAll(void) +{ + DEFiRet; + dbgprintf("All Rulesets:\n"); + llExecFunc(&llRulesets, doDebugPrintAll, NULL); + dbgprintf("End of Rulesets.\n"); + RETiRet; +} + + +/* Create a ruleset-specific "main" queue for this ruleset. If one is already + * defined, an error message is emitted but nothing else is done. + * Note: we use the main message queue parameters for queue creation and access + * syslogd.c directly to obtain these. This is far from being perfect, but + * considered acceptable for the time being. + * rgerhards, 2009-10-27 + */ +static rsRetVal +rulesetCreateQueue(void __attribute__((unused)) *pVal, int *pNewVal) +{ + DEFiRet; + + if(pCurrRuleset == NULL) { + errmsg.LogError(0, RS_RET_NO_CURR_RULESET, "error: currently no specific ruleset specified, thus a " + "queue can not be added to it"); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } + + if(pCurrRuleset->pQueue != NULL) { + errmsg.LogError(0, RS_RET_RULES_QUEUE_EXISTS, "error: ruleset already has a main queue, can not " + "add another one"); + ABORT_FINALIZE(RS_RET_RULES_QUEUE_EXISTS); + } + + if(pNewVal == 0) + FINALIZE; /* if it is turned off, we do not need to change anything ;) */ + + dbgprintf("adding a ruleset-specific \"main\" queue"); + CHKiRet(createMainQueue(&pCurrRuleset->pQueue, UCHAR_CONSTANT("ruleset"))); + +finalize_it: + RETiRet; +} + + +/* Add a ruleset specific parser to the ruleset. Note that adding the first + * parser automatically disables the default parsers. If they are needed as well, + * the must be added via explicit config directives. + * Note: this is the only spot in the code that requires the parser object. In order + * to solve some class init bootstrap sequence problems, we get the object handle here + * instead of during module initialization. Note that objUse() is capable of being + * called multiple times. + * rgerhards, 2009-11-04 + */ +static rsRetVal +rulesetAddParser(void __attribute__((unused)) *pVal, uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + assert(pCurrRuleset != NULL); + + CHKiRet(objUse(parser, CORE_COMPONENT)); + iRet = parser.FindParser(&pParser, pName); + if(iRet == RS_RET_PARSER_NOT_FOUND) { + errmsg.LogError(0, RS_RET_PARSER_NOT_FOUND, "error: parser '%s' unknown at this time " + "(maybe defined too late in rsyslog.conf?)", pName); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } else if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "error trying to find parser '%s'\n", pName); + FINALIZE; + } + + CHKiRet(parser.AddParserToList(&pCurrRuleset->pParserLst, pParser)); + + dbgprintf("added parser '%s' to ruleset '%s'\n", pName, pCurrRuleset->pszName); +RUNLOG_VAR("%p", pCurrRuleset->pParserLst); + +finalize_it: + d_free(pName); /* no longer needed */ + + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(ruleset) +CODESTARTobjQueryInterface(ruleset) + if(pIf->ifVersion != rulesetCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = rulesetConstruct; + pIf->ConstructFinalize = rulesetConstructFinalize; + pIf->Destruct = rulesetDestruct; + pIf->DebugPrint = rulesetDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->DestructAllActions = destructAllActions; + pIf->AddRule = addRule; + pIf->ProcessMsg = processMsg; + pIf->SetName = setName; + pIf->DebugPrintAll = debugPrintAll; + pIf->GetCurrent = GetCurrent; + pIf->GetRuleset = GetRuleset; + pIf->SetDefaultRuleset = SetDefaultRuleset; + pIf->SetCurrRuleset = SetCurrRuleset; + pIf->GetRulesetQueue = GetRulesetQueue; + pIf->GetParserList = GetParserList; +finalize_it: +ENDobjQueryInterface(ruleset) + + +/* Exit the ruleset class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ + llDestroy(&llRulesets); + objRelease(errmsg, CORE_COMPONENT); + objRelease(rule, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); +ENDObjClassExit(ruleset) + + +/* Initialize the ruleset class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(rule, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, rulesetDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, rulesetConstructFinalize); + + /* prepare global data */ + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + + /* config file handlers */ + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetparser", 0, eCmdHdlrGetWord, rulesetAddParser, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetcreatemainqueue", 0, eCmdHdlrBinary, rulesetCreateQueue, NULL, NULL)); +ENDObjClassInit(ruleset) + +/* vi:set ai: + */ diff --git a/runtime/ruleset.h b/runtime/ruleset.h new file mode 100644 index 00000000..222d773e --- /dev/null +++ b/runtime/ruleset.h @@ -0,0 +1,66 @@ +/* The ruleset object. + * + * This implements rulesets within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULESET_H +#define INCLUDED_RULESET_H + +#include "queue.h" +#include "linkedlist.h" + +/* the ruleset object */ +struct ruleset_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ + uchar *pszName; /* name of our ruleset */ + qqueue_t *pQueue; /* "main" message queue, if the ruleset has its own (else NULL) */ + parserList_t *pParserLst;/* list of parsers to use for this ruleset */ +}; + +/* interfaces */ +BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(ruleset); + rsRetVal (*DebugPrintAll)(void); + rsRetVal (*Construct)(ruleset_t **ppThis); + rsRetVal (*ConstructFinalize)(ruleset_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(ruleset_t **ppThis); + rsRetVal (*IterateAllActions)(rsRetVal (*pFunc)(void*, void*), void* pParam); + rsRetVal (*DestructAllActions)(void); + rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); + rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); + rsRetVal (*ProcessMsg)(msg_t *pMsg); + rsRetVal (*GetRuleset)(ruleset_t **ppThis, uchar*); + rsRetVal (*SetDefaultRuleset)(uchar*); + rsRetVal (*SetCurrRuleset)(uchar*); + ruleset_t* (*GetCurrent)(void); + qqueue_t* (*GetRulesetQueue)(ruleset_t*); + /* v3, 2009-11-04 */ + parserList_t* (*GetParserList)(msg_t *); +ENDinterface(ruleset) +#define rulesetCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(ruleset); + +#endif /* #ifndef INCLUDED_RULESET_H */ diff --git a/runtime/srUtils.h b/runtime/srUtils.h index bfce4cbb..c4f73e16 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -92,6 +92,7 @@ void srSleep(int iSeconds, int iuSeconds); char *rs_strerror_r(int errnum, char *buf, size_t buflen); int decodeSyslogName(uchar *name, syslogName_t *codetab); int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); +rsRetVal getFileSize(uchar *pszName, off_t *pSize); /* mutex operations */ /* some macros to cancel-safe lock a mutex (it will automatically be released @@ -108,20 +109,18 @@ int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); #define mutex_cancelsafe_unlock(mut) pthread_cleanup_pop(1) /* some useful constants */ -#define MUTEX_ALREADY_LOCKED 0 -#define LOCK_MUTEX 1 #define DEFVARS_mutexProtection\ - int iCancelStateSave; \ int bLockedOpIsLocked=0 #define BEGIN_MTX_PROTECTED_OPERATIONS(mut, bMustLock) \ if(bMustLock == LOCK_MUTEX) { \ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ d_pthread_mutex_lock(mut); \ + assert(bLockedOpIsLocked == 0); \ bLockedOpIsLocked = 1; \ } #define END_MTX_PROTECTED_OPERATIONS(mut) \ if(bLockedOpIsLocked) { \ d_pthread_mutex_unlock(mut); \ - pthread_setcancelstate(iCancelStateSave, NULL); \ + bLockedOpIsLocked = 0; \ } + #endif diff --git a/runtime/srutils.c b/runtime/srutils.c index d01ca20d..2bed624e 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -46,6 +46,9 @@ #include "srUtils.h" #include "obj.h" +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif /* here we host some syslog specific names. There currently is no better place * to do it, but over here is also not ideal... -- rgerhards, 2008-02-14 @@ -158,7 +161,7 @@ uchar *srUtilStrDup(uchar *pOld, size_t len) assert(pOld != NULL); - if((pNew = malloc(len + 1)) != NULL) + if((pNew = MALLOC(len + 1)) != NULL) memcpy(pNew, pOld, len + 1); return pNew; @@ -183,7 +186,7 @@ int makeFileParentDirs(uchar *szFile, size_t lenFile, mode_t mode, assert(lenFile > 0); len = lenFile + 1; /* add one for '\0'-byte */ - if((pszWork = malloc(sizeof(uchar) * len)) == NULL) + if((pszWork = MALLOC(sizeof(uchar) * len)) == NULL) return -1; memcpy(pszWork, szFile, len); for(p = pszWork+1 ; *p ; p++) @@ -326,7 +329,7 @@ rsRetVal genFileName(uchar **ppName, uchar *pDirName, size_t lenDirName, uchar * } lenName = lenDirName + 1 + lenFName + lenBuf + 1; /* last +1 for \0 char! */ - if((pName = malloc(sizeof(uchar) * lenName)) == NULL) + if((pName = MALLOC(sizeof(uchar) * lenName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); /* got memory, now construct string */ @@ -366,20 +369,34 @@ int getNumberDigits(long lNum) /* compute an absolute time timeout suitable for calls to pthread_cond_timedwait() + * iTimeout is in milliseconds * rgerhards, 2008-01-14 */ rsRetVal timeoutComp(struct timespec *pt, long iTimeout) { +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ + +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, pt); +# else + gettimeofday(&tv, NULL); + pt->tv_sec = tv.tv_sec; + pt->tv_nsec = tv.tv_usec * 1000; +# endif + pt->tv_sec += iTimeout / 1000; pt->tv_nsec += (iTimeout % 1000) * 1000000; /* think INTEGER arithmetic! */ if(pt->tv_nsec > 999999999) { /* overrun? */ pt->tv_nsec -= 1000000000; + ++pt->tv_sec; } - pt->tv_sec += iTimeout / 1000; ENDfunc return RS_RET_OK; /* so far, this is static... */ } @@ -395,11 +412,21 @@ timeoutVal(struct timespec *pt) { struct timespec t; long iTimeout; - BEGINfunc +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif iTimeout = (pt->tv_nsec - t.tv_nsec) / 1000000; iTimeout += (pt->tv_sec - t.tv_sec) * 1000; @@ -553,6 +580,33 @@ int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep) } +/* get the size of a file or return appropriate error code. If an error is returned, + * *pSize content is undefined. + * rgerhards, 2009-06-12 + */ +rsRetVal +getFileSize(uchar *pszName, off_t *pSize) +{ + int ret; + struct stat statBuf; + DEFiRet; + + ret = stat((char*) pszName, &statBuf); + if(ret == -1) { + switch(errno) { + case EACCES: ABORT_FINALIZE(RS_RET_NO_FILE_ACCESS); + case ENOTDIR: + case ENOENT: ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + default: ABORT_FINALIZE(RS_RET_FILE_NO_STAT); + } + } + + *pSize = statBuf.st_size; + +finalize_it: + RETiRet; +} + /* vim:set ai: */ diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..f565dc90 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1,4 +1,3 @@ -//TODO: O_TRUC mode! /* The serial stream class. * * A serial stream provides serial data access. In theory, serial streams @@ -7,8 +6,17 @@ * "driver"). * * File begun on 2008-01-09 by RGerhards + * Large modifications in 2009-06 to support using it with omfile, including zip writer. + * Note that this file obtains the zlib wrapper object is needed, but it never frees it + * again. While this sounds like a leak (and one may argue it actually is), there is no + * harm associated with that. The reason is that strm is a core object, so it is terminated + * only when rsyslogd exists. As we could only release on termination (or else bear more + * overhead for keeping track of how many users we have), not releasing zlibw is OK, because + * it will be released when rsyslogd terminates. We may want to revisit this decision if + * it turns out to be problematic. Then, we need to quasi-refcount the number of accesses + * to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -39,24 +47,198 @@ #include <unistd.h> #include <sys/stat.h> /* required for HP UX */ #include <errno.h> +#include <pthread.h> #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" #include "obj.h" #include "stream.h" +#include "unicode-helper.h" +#include "module-template.h" +#if HAVE_SYS_PRCTL_H +# include <sys/prctl.h> +#endif + +#define inline /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(zlibw) + +/* forward definitions */ +static rsRetVal strmFlush(strm_t *pThis); +static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal strmCloseFile(strm_t *pThis); +static void *asyncWriterThread(void *pPtr); +static rsRetVal doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); + /* methods */ -/* first, we define type-specific handlers. The provide a generic functionality, +/* Try to resolve a size limit situation. This is used to support custom-file size handlers + * for omfile. It first runs the command, and then checks if we are still above the size + * treshold. Note that this works only with single file names, NOT with circular names. + * Note that pszCurrFName can NOT be taken from pThis, because the stream is closed when + * we are called (and that destroys pszCurrFName, as there is NO CURRENT file name!). So + * we need to receive the name as a parameter. + * initially wirtten 2005-06-21, moved to this class & updates 2009-06-01, both rgerhards + */ +static rsRetVal +resolveFileSizeLimit(strm_t *pThis, uchar *pszCurrFName) +{ + uchar *pParams; + uchar *pCmd; + uchar *p; + off_t actualFileSize; + rsRetVal localRet; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + assert(pszCurrFName != NULL); + + if(pThis->pszSizeLimitCmd == NULL) { + ABORT_FINALIZE(RS_RET_NON_SIZELIMITCMD); /* nothing we can do in this case... */ + } + + /* we first check if we have command line parameters. We assume this, + * when we have a space in the program name. If we find it, everything after + * the space is treated as a single argument. + */ + CHKmalloc(pCmd = ustrdup(pThis->pszSizeLimitCmd)); + + for(p = pCmd ; *p && *p != ' ' ; ++p) { + /* JUST SKIP */ + } + + if(*p == ' ') { + *p = '\0'; /* pretend string-end */ + pParams = p+1; + } else + pParams = NULL; + + /* the execProg() below is probably not great, but at least is is + * fairly secure now. Once we change the way file size limits are + * handled, we should also revisit how this command is run (and + * with which parameters). rgerhards, 2007-07-20 + */ + execProg(pCmd, 1, pParams); + + free(pCmd); + + localRet = getFileSize(pszCurrFName, &actualFileSize); + + if(localRet == RS_RET_OK && actualFileSize >= pThis->iSizeLimit) { + ABORT_FINALIZE(RS_RET_SIZELIMITCMD_DIDNT_RESOLVE); /* OK, it didn't work out... */ + } else if(localRet != RS_RET_FILE_NOT_FOUND) { + /* file not found is OK, the command may have moved away the file */ + ABORT_FINALIZE(localRet); + } + +finalize_it: + if(iRet != RS_RET_OK) { + if(iRet == RS_RET_SIZELIMITCMD_DIDNT_RESOLVE) { + DBGPRINTF("file size limit cmd for file '%s' did no resolve situation\n", pszCurrFName); + } else { + DBGPRINTF("file size limit cmd for file '%s' failed with code %d.\n", pszCurrFName, iRet); + } + pThis->bDisabled = 1; + } + + RETiRet; +} + + +/* Check if the file has grown beyond the configured omfile iSizeLimit + * and, if so, initiate processing. + */ +static rsRetVal +doSizeLimitProcessing(strm_t *pThis) +{ + uchar *pszCurrFName = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + ASSERT(pThis->iSizeLimit != 0); + ASSERT(pThis->fd != -1); + + if(pThis->iCurrOffs >= pThis->iSizeLimit) { + /* strmClosefile() destroys the current file name, so we + * need to preserve it. + */ + CHKmalloc(pszCurrFName = ustrdup(pThis->pszCurrFName)); + CHKiRet(strmCloseFile(pThis)); + CHKiRet(resolveFileSizeLimit(pThis, pszCurrFName)); + } + +finalize_it: + free(pszCurrFName); + RETiRet; +} + + +/* now, we define type-specific handlers. The provide a generic functionality, * but for this specific type of strm. The mapping to these handlers happens during * strm construction. Later on, handlers are called by pointers present in the * strm instance object. */ +/* do the physical open() call on a file. + */ +static rsRetVal +doPhysOpen(strm_t *pThis) +{ + int iFlags = 0; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + + /* compute which flags we need to provide to open */ + switch(pThis->tOperationsMode) { + case STREAMMODE_READ: + iFlags = O_CLOEXEC | O_NOCTTY | O_RDONLY; + break; + case STREAMMODE_WRITE: /* legacy mode used inside queue engine */ + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT; + break; + case STREAMMODE_WRITE_TRUNC: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_TRUNC; + break; + case STREAMMODE_WRITE_APPEND: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_APPEND; + break; + default:assert(0); + break; + } + if(pThis->sType == STREAMTYPE_NAMED_PIPE) { + DBGPRINTF("Note: stream '%s' is a named pipe, open with O_NONBLOCK\n", pThis->pszCurrFName); + iFlags |= O_NONBLOCK; + } + + pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); + DBGPRINTF("file '%s' opened as #%d with mode %d\n", pThis->pszCurrFName, pThis->fd, pThis->tOpenMode); + if(pThis->fd == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + dbgoprint((obj_t*) pThis, "open error %d, file '%s': %s\n", errno, pThis->pszCurrFName, errStr); + if(err == ENOENT) + ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + else + ABORT_FINALIZE(RS_RET_IO_ERROR); + } else { + if(!ustrcmp(pThis->pszCurrFName, UCHAR_CONSTANT(_PATH_CONSOLE)) || isatty(pThis->fd)) { + DBGPRINTF("file %d is a tty-type file\n", pThis->fd); + pThis->bIsTTY = 1; + } else { + pThis->bIsTTY = 0; + } + } + +finalize_it: + RETiRet; +} + + /* open a strm file * It is OK to call this function when the stream is already open. In that * case, it returns immediately with RS_RET_OK @@ -64,10 +246,8 @@ DEFobjStaticHelpers static rsRetVal strmOpenFile(strm_t *pThis) { DEFiRet; - int iFlags; ASSERT(pThis != NULL); - ASSERT(pThis->tOperationsMode == STREAMMODE_READ || pThis->tOperationsMode == STREAMMODE_WRITE); if(pThis->fd != -1) ABORT_FINALIZE(RS_RET_OK); @@ -80,7 +260,7 @@ static rsRetVal strmOpenFile(strm_t *pThis) pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); } else { if(pThis->pszDir == NULL) { - if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) + if((pThis->pszCurrFName = ustrdup(pThis->pszFName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } else { CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, @@ -88,34 +268,41 @@ static rsRetVal strmOpenFile(strm_t *pThis) } } - /* compute which flags we need to provide to open */ - if(pThis->tOperationsMode == STREAMMODE_READ) - iFlags = O_RDONLY; - else - iFlags = O_WRONLY | O_CREAT; - - iFlags |= pThis->iAddtlOpenFlags; - - pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); - if(pThis->fd == -1) { - int ierrnoSave = errno; - dbgoprint((obj_t*) pThis, "open error %d, file '%s'\n", errno, pThis->pszCurrFName); - if(ierrnoSave == ENOENT) - ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); - else - ABORT_FINALIZE(RS_RET_IO_ERROR); - } + CHKiRet(doPhysOpen(pThis)); pThis->iCurrOffs = 0; + if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) { + /* we need to obtain the current offset */ + off_t offset; + CHKiRet(getFileSize(pThis->pszCurrFName, &offset)); + pThis->iCurrOffs = offset; + } - dbgoprint((obj_t*) pThis, "opened file '%s' for %s (0x%x) as %d\n", pThis->pszCurrFName, - (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", iFlags, pThis->fd); + dbgoprint((obj_t*) pThis, "opened file '%s' for %s as %d\n", pThis->pszCurrFName, + (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", pThis->fd); finalize_it: RETiRet; } +/* wait for the output writer thread to be done. This must be called before actions + * that require data to be persisted. May be called in non-async mode and is a null + * operation than. Must be called with the mutex locked. + */ +static inline void +strmWaitAsyncWriterDone(strm_t *pThis) +{ + BEGINfunc + if(pThis->bAsyncWrite) { + /* awake writer thread and make it write out everything */ + pthread_cond_signal(&pThis->notEmpty); + d_pthread_cond_wait(&pThis->isEmpty, &pThis->mut); + } + ENDfunc +} + + /* close a strm file * Note that the bDeleteOnClose flag is honored. If it is set, the file will be * deleted after close. This is in support for the qRead thread. @@ -128,14 +315,33 @@ static rsRetVal strmCloseFile(strm_t *pThis) ASSERT(pThis->fd != -1); dbgoprint((obj_t*) pThis, "file %d closing\n", pThis->fd); - if(pThis->tOperationsMode == STREAMMODE_WRITE) - strmFlush(pThis); + if(!pThis->bInClose && pThis->tOperationsMode != STREAMMODE_READ) { + pThis->bInClose = 1; + if(pThis->bAsyncWrite) { + strmFlush(pThis); + } else { + strmWaitAsyncWriterDone(pThis); + } + pThis->bInClose = 0; + } - close(pThis->fd); // TODO: error check + close(pThis->fd); pThis->fd = -1; + if(pThis->fdDir != -1) { + /* close associated directory handle, if it is open */ + close(pThis->fdDir); + pThis->fdDir = -1; + } + if(pThis->bDeleteOnClose) { - unlink((char*) pThis->pszCurrFName); // TODO: check returncode + if(unlink((char*) pThis->pszCurrFName) == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("error %d unlinking '%s' - ignored: %s\n", + errno, pThis->pszCurrFName, errStr); + } } pThis->iCurrOffs = 0; /* we are back at begin of file */ @@ -229,15 +435,12 @@ strmHandleEOF(strm_t *pThis) ISOBJ_TYPE_assert(pThis, strm); switch(pThis->sType) { case STREAMTYPE_FILE_SINGLE: + case STREAMTYPE_NAMED_PIPE: ABORT_FINALIZE(RS_RET_EOF); break; case STREAMTYPE_FILE_CIRCULAR: /* we have multiple files and need to switch to the next one */ /* TODO: think about emulating EOF in this case (not yet needed) */ -#if 0 - if(pThis->iMaxFiles == 0) /* TODO: why do we need this? ;) */ - ABORT_FINALIZE(RS_RET_EOF); -#endif dbgoprint((obj_t*) pThis, "file %d EOF\n", pThis->fd); CHKiRet(strmNextFile(pThis)); break; @@ -295,7 +498,7 @@ finalize_it: * NOTE: needs to be enhanced to support sticking with a strm entry (if not * deleted). */ -rsRetVal strmReadChar(strm_t *pThis, uchar *pC) +static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) { DEFiRet; @@ -329,7 +532,7 @@ finalize_it: * character buffering capability. * rgerhards, 2008-01-07 */ -rsRetVal strmUnreadChar(strm_t *pThis, uchar c) +static rsRetVal strmUnreadChar(strm_t *pThis, uchar c) { ASSERT(pThis != NULL); ASSERT(pThis->iUngetC == -1); @@ -351,7 +554,7 @@ rsRetVal strmUnreadChar(strm_t *pThis, uchar c) * are pthread_killed() upon termination. So if we use their native pointer, they * can cleanup (but only then). */ -rsRetVal +static rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr) { DEFiRet; @@ -360,19 +563,19 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr) ASSERT(pThis != NULL); ASSERT(ppCStr != NULL); - CHKiRet(rsCStrConstruct(ppCStr)); + CHKiRet(cstrConstruct(ppCStr)); /* now read the line */ CHKiRet(strmReadChar(pThis, &c)); while(c != '\n') { - CHKiRet(rsCStrAppendChar(*ppCStr, c)); + CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } - CHKiRet(rsCStrFinish(*ppCStr)); + CHKiRet(cstrFinalize(*ppCStr)); finalize_it: if(iRet != RS_RET_OK && *ppCStr != NULL) - rsCStrDestruct(ppCStr); + cstrDestruct(ppCStr); RETiRet; } @@ -383,26 +586,75 @@ finalize_it: BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! */ pThis->iCurrFNum = 1; pThis->fd = -1; + pThis->fdDir = -1; pThis->iUngetC = -1; pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); - pThis->tOpenMode = 0600; /* TODO: make configurable */ + pThis->tOpenMode = 0600; ENDobjConstruct(strm) /* ConstructionFinalizer * rgerhards, 2008-01-09 */ -rsRetVal strmConstructFinalize(strm_t *pThis) +static rsRetVal strmConstructFinalize(strm_t *pThis) { + rsRetVal localRet; + int i; DEFiRet; ASSERT(pThis != NULL); - if(pThis->pIOBuf == NULL) { /* allocate our io buffer in case we have not yet */ - if((pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - pThis->iBufPtrMax = 0; /* results in immediate read request */ + pThis->iBufPtrMax = 0; /* results in immediate read request */ + if(pThis->iZipLevel) { /* do we need a zip buf? */ + localRet = objUse(zlibw, LM_ZLIBW_FILENAME); + if(localRet != RS_RET_OK) { + pThis->iZipLevel = 0; + DBGPRINTF("stream was requested with zip mode, but zlibw module unavailable (%d) - using " + "without zip\n", localRet); + } else { + /* we use the same size as the original buf, as we would like + * to make sure we can write out everything with a SINGLE api call! + * We add another 128 bytes to take care of the gzip header and "all eventualities". + */ + CHKmalloc(pThis->pZipBuf = (Bytef*) MALLOC(sizeof(uchar) * pThis->sIOBufSize + 128)); + } + } + + /* if we are aset to sync, we must obtain a file handle to the directory for fsync() purposes */ + if(pThis->bSync && !pThis->bIsTTY) { + pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY | O_CLOEXEC | O_NOCTTY); + if(pThis->fdDir == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("error %d opening directory file for fsync() use - fsync for directory disabled: %s\n", + errno, errStr); + } + } + + /* if we have a flush interval, we need to do async writes in any case */ + if(pThis->iFlushInterval != 0) { + pThis->bAsyncWrite = 1; + } + + /* if we work asynchronously, we need a couple of synchronization objects */ + if(pThis->bAsyncWrite) { + pthread_mutex_init(&pThis->mut, 0); + pthread_cond_init(&pThis->notFull, 0); + pthread_cond_init(&pThis->notEmpty, 0); + pthread_cond_init(&pThis->isEmpty, 0); + pThis->iCnt = pThis->iEnq = pThis->iDeq = 0; + for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) { + CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); + } + pThis->pIOBuf = pThis->asyncBuf[0].pBuf; + pThis->bStopWriter = 0; + if(pthread_create(&pThis->writerThreadID, NULL, asyncWriterThread, pThis) != 0) + DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis); + } else { + /* we work synchronously, so we need to alloc a fixed pIOBuf */ + CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); } finalize_it: @@ -410,24 +662,57 @@ finalize_it: } +/* stop the writer thread (we MUST be runnnig asynchronously when this method + * is called!). Note that the mutex must be locked! -- rgerhards, 2009-07-06 + */ +static inline void +stopWriter(strm_t *pThis) +{ + BEGINfunc + pThis->bStopWriter = 1; + pthread_cond_signal(&pThis->notEmpty); + d_pthread_mutex_unlock(&pThis->mut); + pthread_join(pThis->writerThreadID, NULL); + ENDfunc +} + + /* destructor for the strm object */ BEGINobjDestruct(strm) /* be sure to specify the object type also in END and CODESTART macros! */ + int i; CODESTARTobjDestruct(strm) - if(pThis->tOperationsMode == STREAMMODE_WRITE) + if(pThis->bAsyncWrite) + /* Note: mutex will be unlocked in stopWriter! */ + d_pthread_mutex_lock(&pThis->mut); + + if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); - /* ... then free resources */ + if(pThis->bAsyncWrite) { + stopWriter(pThis); + pthread_mutex_destroy(&pThis->mut); + pthread_cond_destroy(&pThis->notFull); + pthread_cond_destroy(&pThis->notEmpty); + pthread_cond_destroy(&pThis->isEmpty); + for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) { + free(pThis->asyncBuf[i].pBuf); + } + } else { + free(pThis->pIOBuf); + } + + /* Finally, we can free the resources. + * IMPORTANT: we MUST free this only AFTER the ansyncWriter has been stopped, else + * we get random errors... + */ if(pThis->fd != -1) strmCloseFile(pThis); - if(pThis->pszDir != NULL) - free(pThis->pszDir); - if(pThis->pIOBuf != NULL) - free(pThis->pIOBuf); - if(pThis->pszCurrFName != NULL) - free(pThis->pszCurrFName); - if(pThis->pszFName != NULL) - free(pThis->pszFName); + free(pThis->pszDir); + free(pThis->pZipBuf); + free(pThis->pszCurrFName); + free(pThis->pszFName); + ENDobjDestruct(strm) @@ -443,6 +728,9 @@ static rsRetVal strmCheckNextOutputFile(strm_t *pThis) if(pThis->fd == -1) FINALIZE; + /* wait for output to be empty, so that our counts are correct */ + strmWaitAsyncWriterDone(pThis); + if(pThis->iCurrOffs >= pThis->iMaxFileSize) { dbgoprint((obj_t*) pThis, "max file size %ld reached for %d, now %ld - starting new file\n", (long) pThis->iMaxFileSize, pThis->fd, (long) pThis->iCurrOffs); @@ -453,47 +741,375 @@ finalize_it: RETiRet; } + +/* try to recover a tty after a write error. This may have happend + * due to vhangup(), and, if so, we can simply re-open it. + */ +#ifdef linux +# define ERR_TTYHUP EIO +#else +# define ERR_TTYHUP EBADF +#endif +static rsRetVal +tryTTYRecover(strm_t *pThis, int err) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + if(err == ERR_TTYHUP) { + close(pThis->fd); + CHKiRet(doPhysOpen(pThis)); + } + +finalize_it: + RETiRet; +} +#undef ER_TTYHUP + + +/* issue write() api calls until either the buffer is completely + * written or an error occured (it may happen that multiple writes + * are required, what is perfectly legal. On exit, *pLenBuf contains + * the number of bytes actually written. + * rgerhards, 2009-06-08 + */ +static rsRetVal +doWriteCall(strm_t *pThis, uchar *pBuf, size_t *pLenBuf) +{ + ssize_t lenBuf; + ssize_t iTotalWritten; + ssize_t iWritten; + char *pWriteBuf; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + + lenBuf = *pLenBuf; + pWriteBuf = (char*) pBuf; + iTotalWritten = 0; + do { + iWritten = write(pThis->fd, pWriteBuf, lenBuf); + if(iWritten < 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("log file (%d) write error %d: %s\n", pThis->fd, err, errStr); + if(err == EINTR) { + /*NO ERROR, just continue */; + } else { + if(pThis->bIsTTY) { + CHKiRet(tryTTYRecover(pThis, err)); + } else { + ABORT_FINALIZE(RS_RET_IO_ERROR); + /* Would it make sense to cover more error cases? So far, I + * do not see good reason to do so. + */ + } + } + } + /* advance buffer to next write position */ + iTotalWritten += iWritten; + lenBuf -= iWritten; + pWriteBuf += iWritten; + } while(lenBuf > 0); /* Warning: do..while()! */ + + dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, (int) iWritten); + +finalize_it: + *pLenBuf = iTotalWritten; + RETiRet; +} + + + /* write memory buffer to a stream object. - * To support direct writes of large objects, this method may be called - * with a buffer pointing to some region other than the stream buffer itself. - * However, in that case the stream buffer must be empty (strmFlush() has to - * be called before), because we would otherwise mess up with the sequence - * inside the stream. -- rgerhards, 2008-01-10 */ -static rsRetVal strmWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static inline rsRetVal +doWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + DEFiRet; + + ASSERT(pThis != NULL); + + if(pThis->iZipLevel) { + CHKiRet(doZipWrite(pThis, pBuf, lenBuf)); + } else { + /* write without zipping */ + CHKiRet(strmPhysWrite(pThis, pBuf, lenBuf)); + } + +finalize_it: + RETiRet; +} + + +/* This function is called to "do" an async write call, what primarily means that + * the data is handed over to the writer thread (which will then do the actual write + * in parallel). Note that the stream mutex has already been locked by the + * strmWrite...() calls. Also note that we always have only a single producer, + * so we can simply serially assign the next free buffer to it and be sure that + * the very some producer comes back in sequence to submit the then-filled buffers. + * This also enables us to timout on partially written buffers. -- rgerhards, 2009-07-06 + */ +static inline rsRetVal +doAsyncWriteInternal(strm_t *pThis, size_t lenBuf) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + + while(pThis->iCnt >= STREAM_ASYNC_NUMBUFS) + d_pthread_cond_wait(&pThis->notFull, &pThis->mut); + + pThis->asyncBuf[pThis->iEnq % STREAM_ASYNC_NUMBUFS].lenBuf = lenBuf; + pThis->pIOBuf = pThis->asyncBuf[++pThis->iEnq % STREAM_ASYNC_NUMBUFS].pBuf; + + pThis->bDoTimedWait = 0; /* everything written, no need to timeout partial buffer writes */ + if(++pThis->iCnt == 1) + pthread_cond_signal(&pThis->notEmpty); + + RETiRet; +} + + +/* schedule writing to the stream. Depending on our concurrency settings, + * this either directly writes to the stream or schedules writing via + * the background thread. -- rgerhards, 2009-07-07 + */ +static rsRetVal +strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; - int iWritten; ASSERT(pThis != NULL); - ASSERT(pBuf == pThis->pIOBuf || pThis->iBufPtr == 0); + + if(pThis->bAsyncWrite) { + CHKiRet(doAsyncWriteInternal(pThis, lenBuf)); + } else { + CHKiRet(doWriteInternal(pThis, pBuf, lenBuf)); + } + + pThis->iBufPtr = 0; /* we are at the begin of a new buffer */ + +finalize_it: + RETiRet; +} + + + +/* This is the writer thread for asynchronous mode. + * -- rgerhards, 2009-07-06 + */ +static void* +asyncWriterThread(void *pPtr) +{ + int iDeq; + struct timespec t; + sbool bTimedOut = 0; + strm_t *pThis = (strm_t*) pPtr; + ISOBJ_TYPE_assert(pThis, strm); + + BEGINfunc +# if HAVE_PRCTL && defined PR_SET_NAME + if(prctl(PR_SET_NAME, "rs:asyn strmwr", 0, 0, 0) != 0) { + DBGPRINTF("prctl failed, not setting thread name for '%s'\n", "stream writer"); + } +#endif + + while(1) { /* loop broken inside */ + d_pthread_mutex_lock(&pThis->mut); + while(pThis->iCnt == 0) { + if(pThis->bStopWriter) { + pthread_cond_broadcast(&pThis->isEmpty); + d_pthread_mutex_unlock(&pThis->mut); + goto finalize_it; /* break main loop */ + } + if(bTimedOut && pThis->iBufPtr > 0) { + /* if we timed out, we need to flush pending data */ + strmFlush(pThis); + bTimedOut = 0; + continue; /* now we should have data */ + } + bTimedOut = 0; + timeoutComp(&t, pThis->iFlushInterval * 2000); /* *1000 millisconds */ + if(pThis->bDoTimedWait) { + if(pthread_cond_timedwait(&pThis->notEmpty, &pThis->mut, &t) != 0) { + int err = errno; + if(err == ETIMEDOUT) { + bTimedOut = 1; + } else { + bTimedOut = 1; + char errStr[1024]; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("stream async writer timeout with error (%d): %s - ignoring\n", + err, errStr); + } + } + } else { + d_pthread_cond_wait(&pThis->notEmpty, &pThis->mut); + } + } + + bTimedOut = 0; /* we may have timed out, but there *is* work to do... */ + + iDeq = pThis->iDeq++ % STREAM_ASYNC_NUMBUFS; + doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf); + // TODO: error check????? 2009-07-06 + + --pThis->iCnt; + if(pThis->iCnt < STREAM_ASYNC_NUMBUFS) { + pthread_cond_signal(&pThis->notFull); + if(pThis->iCnt == 0) + pthread_cond_broadcast(&pThis->isEmpty); + } + d_pthread_mutex_unlock(&pThis->mut); + } + +finalize_it: + ENDfunc + return NULL; /* to keep pthreads happy */ +} + + +/* sync the file to disk, so that any unwritten data is persisted. This + * also syncs the directory and thus makes sure that the file survives + * fatal failure. Note that we do NOT return an error status if the + * sync fails. Doing so would probably cause more trouble than it + * is worth (read: data loss may occur where we otherwise might not + * have it). -- rgerhards, 2009-06-08 + */ +#undef SYNCCALL +#if HAVE_FDATASYNC +# define SYNCCALL(x) fdatasync(x) +#else +# define SYNCCALL(x) fsync(x) +#endif +static rsRetVal +syncFile(strm_t *pThis) +{ + int ret; + DEFiRet; + + if(pThis->bIsTTY) + FINALIZE; /* TTYs can not be synced */ + + DBGPRINTF("syncing file %d\n", pThis->fd); + ret = SYNCCALL(pThis->fd); + if(ret != 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("sync failed for file %d with error (%d): %s - ignoring\n", + pThis->fd, err, errStr); + } + + if(pThis->fdDir != -1) { + ret = fsync(pThis->fdDir); + } + +finalize_it: + RETiRet; +} +#undef SYNCCALL + +/* physically write to the output file. the provided data is ready for + * writing (e.g. zipped if we are requested to do that). + * Note that if the write() API fails, we do not reset any pointers, but return + * an error code. That means we may redo work in the next iteration. + * rgerhards, 2009-06-04 + */ +static rsRetVal +strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + size_t iWritten; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); - iWritten = write(pThis->fd, pBuf, lenBuf); - dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, iWritten); - /* TODO: handle error case -- rgerhards, 2008-01-07 */ - - /* Now indicate buffer empty again. We do this in any case, because there - * is no way we could react more intelligently to an error during write. - * This MUST be done BEFORE strCheckNextOutputFile(), otherwise we have an - * endless loop. We reset the buffer pointer also in finalize_it - this is - * necessary if we run into problems. Not resetting it would again cause an - * endless loop. So it is better to loose some data (which also justifies - * duplicating that code, too...) -- rgerhards, 2008-01-10 - */ - pThis->iBufPtr = 0; + iWritten = lenBuf; + CHKiRet(doWriteCall(pThis, pBuf, &iWritten)); + pThis->iCurrOffs += iWritten; /* update user counter, if provided */ if(pThis->pUsrWCntr != NULL) *pThis->pUsrWCntr += iWritten; - if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) + if(pThis->bSync) { + CHKiRet(syncFile(pThis)); + } + + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { CHKiRet(strmCheckNextOutputFile(pThis)); + } else if(pThis->iSizeLimit != 0) { + CHKiRet(doSizeLimitProcessing(pThis)); + } finalize_it: - pThis->iBufPtr = 0; /* see comment above */ + RETiRet; +} + + +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + * Note that we always do a full deflateInit ... deflate ... deflateEnd + * sequence. While this is not optimal, we need to do it because we need + * to ensure that the file is readable even when we are aborted. Doing the + * full sequence brings us as far towards this goal as possible (and not + * doing it would be a total failure). It may be worth considering to + * add a config switch so that the user can decide the risk he is ready + * to take, but so far this is not yet implemented (not even requested ;)). + * rgerhards, 2009-06-04 + * For the time being, we take a very conservative approach and do not run this + * method multithreaded. This is done in an effort to solve a segfault condition + * that seems to be related to the zip code. -- rgerhards, 2009-09-22 + * TODO: make multithreaded again! + */ +static rsRetVal +doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + z_stream zstrm; + int zRet; /* zlib return state */ + sbool bzInitDone = FALSE; + DEFiRet; + assert(pThis != NULL); + assert(pBuf != NULL); + + /* allocate deflate state */ + zstrm.zalloc = Z_NULL; + zstrm.zfree = Z_NULL; + zstrm.opaque = Z_NULL; + zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateInit2()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + bzInitDone = TRUE; + + /* now doing the compression */ + zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ + zstrm.avail_in = lenBuf; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); + zstrm.avail_out = pThis->sIOBufSize; + zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + if(zstrm.avail_out == pThis->sIOBufSize) + break; /* this is valid, indicates end of compression --> see zlib howto */ + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, pThis->sIOBufSize - zstrm.avail_out)); + } while (zstrm.avail_out == 0); + assert(zstrm.avail_in == 0); /* all input will be used */ + +finalize_it: + if(bzInitDone) { + zRet = zlibw.DeflateEnd(&zstrm); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); + } + } RETiRet; } @@ -503,15 +1119,16 @@ finalize_it: * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 */ -rsRetVal strmFlush(strm_t *pThis) +static rsRetVal +strmFlush(strm_t *pThis) { DEFiRet; ASSERT(pThis != NULL); dbgoprint((obj_t*) pThis, "file %d flush, buflen %ld\n", pThis->fd, (long) pThis->iBufPtr); - if(pThis->tOperationsMode == STREAMMODE_WRITE && pThis->iBufPtr > 0) { - iRet = strmWriteInternal(pThis, pThis->pIOBuf, pThis->iBufPtr); + if(pThis->tOperationsMode != STREAMMODE_READ && pThis->iBufPtr > 0) { + iRet = strmSchedWrite(pThis, pThis->pIOBuf, pThis->iBufPtr); } RETiRet; @@ -545,7 +1162,7 @@ static rsRetVal strmSeek(strm_t *pThis, off_t offs) /* seek to current offset. This is primarily a helper to readjust the OS file * pointer after a strm object has been deserialized. */ -rsRetVal strmSeekCurrOffs(strm_t *pThis) +static rsRetVal strmSeekCurrOffs(strm_t *pThis) { DEFiRet; @@ -558,12 +1175,18 @@ rsRetVal strmSeekCurrOffs(strm_t *pThis) /* write a *single* character to a stream object -- rgerhards, 2008-01-10 */ -rsRetVal strmWriteChar(strm_t *pThis, uchar c) +static rsRetVal strmWriteChar(strm_t *pThis, uchar c) { DEFiRet; ASSERT(pThis != NULL); + if(pThis->bAsyncWrite) + d_pthread_mutex_lock(&pThis->mut); + + if(pThis->bDisabled) + ABORT_FINALIZE(RS_RET_STREAM_DISABLED); + /* if the buffer is full, we need to flush before we can write */ if(pThis->iBufPtr == pThis->sIOBufSize) { CHKiRet(strmFlush(pThis)); @@ -573,12 +1196,19 @@ rsRetVal strmWriteChar(strm_t *pThis, uchar c) pThis->iBufPtr++; finalize_it: + if(pThis->bAsyncWrite) + d_pthread_mutex_unlock(&pThis->mut); + RETiRet; } -/* write an integer value (actually a long) to a stream object */ -rsRetVal strmWriteLong(strm_t *pThis, long i) +/* write an integer value (actually a long) to a stream object + * Note that we do not need to lock the mutex here, because we call + * strmWrite(), which does the lock (aka: we must not lock it, else we + * would run into a recursive lock, resulting in a deadlock!) + */ +static rsRetVal strmWriteLong(strm_t *pThis, long i) { DEFiRet; uchar szBuf[32]; @@ -593,45 +1223,67 @@ finalize_it: } -/* write memory buffer to a stream object +/* write memory buffer to a stream object. + * process the data in chunks and copy it over to our buffer. The caller-provided data + * may theoritically be larger than our buffer. In that case, we do multiple copies. One + * may argue if it were more efficient to write out the caller-provided buffer in that case + * and earlier versions of rsyslog did this. However, this introduces a lot of complexity + * inside the buffered writer and potential performance bottlenecks when trying to solve + * it. Now keep in mind that we actually do (almost?) never have a case where the + * caller-provided buffer is larger than our one. So instead of optimizing a case + * which normally does not exist, we expect some degradation in its case but make us + * perform better in the regular cases. -- rgerhards, 2009-07-07 */ -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static rsRetVal +strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; - size_t iPartial; + size_t iWrite; + size_t iOffset; ASSERT(pThis != NULL); ASSERT(pBuf != NULL); - /* check if the to-be-written data is larger than our buffer size */ - if(lenBuf >= pThis->sIOBufSize) { - /* it is - so we do a direct write, that is most efficient. - * TODO: is it really? think about disk block sizes! - */ - CHKiRet(strmFlush(pThis)); /* we need to flush first!!! */ - CHKiRet(strmWriteInternal(pThis, pBuf, lenBuf)); - } else { - /* data fits into a buffer - we just need to see if it - * fits into the current buffer... - */ - if(pThis->iBufPtr + lenBuf > pThis->sIOBufSize) { - /* nope, so we must split it */ - iPartial = pThis->sIOBufSize - pThis->iBufPtr; /* this fits in current buf */ - if(iPartial > 0) { /* the buffer was exactly full, can not write anything! */ - memcpy(pThis->pIOBuf + pThis->iBufPtr, pBuf, iPartial); - pThis->iBufPtr += iPartial; - } +//DBGPRINTF("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n", pThis, pBuf,lenBuf, pThis->bDisabled, pThis->iSizeLimit, pThis->iCurrOffs); + if(pThis->bAsyncWrite) + d_pthread_mutex_lock(&pThis->mut); + + if(pThis->bDisabled) + ABORT_FINALIZE(RS_RET_STREAM_DISABLED); + + iOffset = 0; + do { + if(pThis->iBufPtr == pThis->sIOBufSize) { CHKiRet(strmFlush(pThis)); /* get a new buffer for rest of data */ - memcpy(pThis->pIOBuf, pBuf + iPartial, lenBuf - iPartial); - pThis->iBufPtr = lenBuf - iPartial; - } else { - /* we have space, so we simply copy over the string */ - memcpy(pThis->pIOBuf + pThis->iBufPtr, pBuf, lenBuf); - pThis->iBufPtr += lenBuf; } + iWrite = pThis->sIOBufSize - pThis->iBufPtr; /* this fits in current buf */ + if(iWrite > lenBuf) + iWrite = lenBuf; + memcpy(pThis->pIOBuf + pThis->iBufPtr, pBuf + iOffset, iWrite); + pThis->iBufPtr += iWrite; + iOffset += iWrite; + lenBuf -= iWrite; + } while(lenBuf > 0); + + /* now check if the buffer right at the end of the write is full and, if so, + * write it. This seems more natural than waiting (hours?) for the next message... + */ + if(pThis->iBufPtr == pThis->sIOBufSize) { + CHKiRet(strmFlush(pThis)); /* get a new buffer for rest of data */ } finalize_it: + if(pThis->bAsyncWrite) { + if(pThis->bDoTimedWait == 0) { + /* we potentially have a partial buffer, so re-activate the + * writer thread that it can set and pick up timeouts. + */ + pThis->bDoTimedWait = 1; + pthread_cond_signal(&pThis->notEmpty); + } + d_pthread_mutex_unlock(&pThis->mut); + } + RETiRet; } @@ -644,34 +1296,27 @@ DEFpropSetMeth(strm, iFileNumDigits, int) DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) - -rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) +DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, bSync, int) +DEFpropSetMeth(strm, sIOBufSize, size_t) +DEFpropSetMeth(strm, iSizeLimit, off_t) +DEFpropSetMeth(strm, iFlushInterval, int) +DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) + +static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { pThis->iMaxFiles = iNewVal; pThis->iFileNumDigits = getNumberDigits(iNewVal); return RS_RET_OK; } -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) -{ - DEFiRet; - - if(iNewVal & O_APPEND) - ABORT_FINALIZE(RS_RET_PARAM_ERROR); - - pThis->iAddtlOpenFlags = iNewVal; - -finalize_it: - RETiRet; -} - /* set the stream's file prefix * The passed-in string is duplicated. So if the caller does not need * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) { DEFiRet; @@ -685,7 +1330,7 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) if(pThis->pszFName != NULL) free(pThis->pszFName); - if((pThis->pszFName = malloc(sizeof(uchar) * iLenName + 1)) == NULL) + if((pThis->pszFName = MALLOC(sizeof(uchar) * (iLenName + 1))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */ @@ -701,7 +1346,7 @@ finalize_it: * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) { DEFiRet; @@ -712,7 +1357,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - if((pThis->pszDir = malloc(sizeof(uchar) * iLenDir + 1)) == NULL) + if((pThis->pszDir = MALLOC(sizeof(uchar) * iLenDir + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDir, pszDir, iLenDir + 1); /* always think about the \0! */ @@ -745,7 +1390,7 @@ finalize_it: * * rgerhards, 2008-01-10 */ -rsRetVal strmRecordBegin(strm_t *pThis) +static rsRetVal strmRecordBegin(strm_t *pThis) { ASSERT(pThis != NULL); ASSERT(pThis->bInRecord == 0); @@ -753,7 +1398,7 @@ rsRetVal strmRecordBegin(strm_t *pThis) return RS_RET_OK; } -rsRetVal strmRecordEnd(strm_t *pThis) +static rsRetVal strmRecordEnd(strm_t *pThis) { DEFiRet; ASSERT(pThis != NULL); @@ -775,7 +1420,7 @@ rsRetVal strmRecordEnd(strm_t *pThis) * We do not serialize the dynamic properties. * rgerhards, 2008-01-10 */ -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) +static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) { DEFiRet; int i; @@ -811,6 +1456,46 @@ finalize_it: } +/* duplicate a stream object excluding dynamic properties. This function is + * primarily meant to provide a duplicate that later on can be used to access + * the data. This is needed, for example, for a restart of the disk queue. + * Note that ConstructFinalize() is NOT called. So our caller may change some + * properties before finalizing things. + * rgerhards, 2009-05-26 + */ +rsRetVal +strmDup(strm_t *pThis, strm_t **ppNew) +{ + strm_t *pNew = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + assert(ppNew != NULL); + + CHKiRet(strmConstruct(&pNew)); + pNew->sType = pThis->sType; + pNew->iCurrFNum = pThis->iCurrFNum; + CHKmalloc(pNew->pszFName = ustrdup(pThis->pszFName)); + pNew->lenFName = pThis->lenFName; + CHKmalloc(pNew->pszDir = ustrdup(pThis->pszDir)); + pNew->lenDir = pThis->lenDir; + pNew->tOperationsMode = pThis->tOperationsMode; + pNew->tOpenMode = pThis->tOpenMode; + pNew->iMaxFileSize = pThis->iMaxFileSize; + pNew->iMaxFiles = pThis->iMaxFiles; + pNew->iFileNumDigits = pThis->iFileNumDigits; + pNew->bDeleteOnClose = pThis->bDeleteOnClose; + pNew->iCurrOffs = pThis->iCurrOffs; + + *ppNew = pNew; + pNew = NULL; + +finalize_it: + if(pNew != NULL) + strmDestruct(&pNew); + + RETiRet; +} /* set a user write-counter. This counter is initialized to zero and * receives the number of bytes written. It is accurate only after a @@ -821,7 +1506,7 @@ finalize_it: * any new set overwrites the previous one. * rgerhards, 2008-02-27 */ -rsRetVal +static rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt) { DEFiRet; @@ -841,8 +1526,8 @@ strmSetWCntr(strm_t *pThis, number_t *pWCnt) /* This function can be used as a generic way to set properties. * rgerhards, 2008-01-11 */ -#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) -rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) +#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, UCHAR_CONSTANT(name), sizeof(name) - 1) +static rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) { DEFiRet; @@ -881,7 +1566,7 @@ finalize_it: * reported on the second call may actually be lower than on the first call. This is due to * file circulation. A caller must deal with that. -- rgerhards, 2008-01-30 */ -rsRetVal +static rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs) { DEFiRet; @@ -909,8 +1594,39 @@ CODESTARTobjQueryInterface(strm) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - /*xxxpIf->oID = OBJvm; SAMPLE */ - + pIf->Construct = strmConstruct; + pIf->ConstructFinalize = strmConstructFinalize; + pIf->Destruct = strmDestruct; + pIf->ReadChar = strmReadChar; + pIf->UnreadChar = strmUnreadChar; + pIf->ReadLine = strmReadLine; + pIf->SeekCurrOffs = strmSeekCurrOffs; + pIf->Write = strmWrite; + pIf->WriteChar = strmWriteChar; + pIf->WriteLong = strmWriteLong; + pIf->SetFName = strmSetFName; + pIf->SetDir = strmSetDir; + pIf->Flush = strmFlush; + pIf->RecordBegin = strmRecordBegin; + pIf->RecordEnd = strmRecordEnd; + pIf->Serialize = strmSerialize; + pIf->GetCurrOffset = strmGetCurrOffset; + pIf->Dup = strmDup; + pIf->SetWCntr = strmSetWCntr; + /* set methods */ + pIf->SetbDeleteOnClose = strmSetbDeleteOnClose; + pIf->SetiMaxFileSize = strmSetiMaxFileSize; + pIf->SetiMaxFiles = strmSetiMaxFiles; + pIf->SetiFileNumDigits = strmSetiFileNumDigits; + pIf->SettOperationsMode = strmSettOperationsMode; + pIf->SettOpenMode = strmSettOpenMode; + pIf->SetsType = strmSetsType; + pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetbSync = strmSetbSync; + pIf->SetsIOBufSize = strmSetsIOBufSize; + pIf->SetiSizeLimit = strmSetiSizeLimit; + pIf->SetiFlushInterval = strmSetiFlushInterval; + pIf->SetpszSizeLimitCmd = strmSetpszSizeLimitCmd; finalize_it: ENDobjQueryInterface(strm) @@ -927,7 +1643,5 @@ BEGINObjClassInit(strm, 1, OBJ_IS_CORE_MODULE) OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmConstructFinalize); ENDObjClassInit(strm) - -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stream.h b/runtime/stream.h index 371358ab..5e3324c5 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -19,7 +19,29 @@ * can easily be persistet. The bottom line is that it makes much sense to * use this class whereever possible as its features may grow in the future. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * An important note on writing gzip format via zlib (kept anonymous + * by request): + * + * -------------------------------------------------------------------------- + * We'd like to make sure the output file is in full gzip format + * (compatible with gzip -d/zcat etc). There is a flag in how the output + * is initialized within zlib to properly add the gzip wrappers to the + * output. (gzip is effectively a small metadata wrapper around raw + * zstream output.) + * + * I had written an old bit of code to do this - the documentation on + * deflatInit2() was pretty tricky to nail down on this specific feature: + * + * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, + * int memLevel, int strategy); + * + * I believe "31" would be the value for the "windowBits" field that you'd + * want to try: + * + * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + * -------------------------------------------------------------------------- + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -47,20 +69,26 @@ #include "obj-types.h" #include "glbl.h" #include "stream.h" +#include "zlibw.h" +#include "apc.h" /* stream types */ typedef enum { STREAMTYPE_FILE_SINGLE = 0, /**< read a single file */ STREAMTYPE_FILE_CIRCULAR = 1, /**< circular files */ - STREAMTYPE_FILE_MONITOR = 2 /**< monitor a (third-party) file */ + STREAMTYPE_FILE_MONITOR = 2, /**< monitor a (third-party) file */ + STREAMTYPE_NAMED_PIPE = 3 /**< file is a named pipe (so far, tested for output only) */ } strmType_t; -typedef enum { +typedef enum { /* when extending, do NOT change existing modes! */ STREAMMMODE_INVALID = 0, STREAMMODE_READ = 1, - STREAMMODE_WRITE = 2 + STREAMMODE_WRITE = 2, + STREAMMODE_WRITE_TRUNC = 3, + STREAMMODE_WRITE_APPEND = 4 } strmMode_t; +#define STREAM_ASYNC_NUMBUFS 2 /* must be a power of 2 -- TODO: make configurable */ /* The strm_t data structure */ typedef struct strm_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ @@ -71,61 +99,96 @@ typedef struct strm_s { int lenFName; strmMode_t tOperationsMode; mode_t tOpenMode; - int iAddtlOpenFlags; /* can be used to specifiy additional (compatible!) open flags */ int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ - int bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ + sbool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ - size_t sIOBufSize;/* size of IO buffer */ + sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ + sbool bSync; /* sync this file after every write? */ + size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; int fd; /* the file descriptor, -1 if closed */ + int fdDir; /* the directory's descriptor, in case bSync is requested (-1 if closed) */ uchar *pszCurrFName; /* name of current file (if open) */ - uchar *pIOBuf; /* io Buffer */ + uchar *pIOBuf; /* the iobuffer currently in use to gather data */ size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ - int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + sbool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + sbool bInClose; /* used to break "deadly close loops", tells us we are already inside a close */ + int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ + Bytef *pZipBuf; + /* support for async flush procesing */ + sbool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ + sbool bStopWriter; /* shall writer thread terminate? */ + sbool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ + int iFlushInterval; /* flush in which interval - 0, no flushing */ + apc_id_t apcID; /* id of current Apc request (used for cancelling) */ + pthread_mutex_t mut;/* mutex for flush in async mode */ + pthread_cond_t notFull; + pthread_cond_t notEmpty; + pthread_cond_t isEmpty; + unsigned short iEnq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ + unsigned short iDeq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ + short iCnt; /* current nbr of elements in buffer */ + struct { + uchar *pBuf; + size_t lenBuf; + } asyncBuf[STREAM_ASYNC_NUMBUFS]; + pthread_t writerThreadID; + int apcRequested; /* is an apc Requested? */ + /* support for omfile size-limiting commands, special counters, NOT persisted! */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ + sbool bIsTTY; /* is this a tty file? */ } strm_t; + /* interfaces */ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(strm_t **ppThis); + rsRetVal (*ConstructFinalize)(strm_t *pThis); + rsRetVal (*Destruct)(strm_t **ppThis); + rsRetVal (*SetMaxFileSize)(strm_t *pThis, int64 iMaxFileSize); + rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); + rsRetVal (*ReadChar)(strm_t *pThis, uchar *pC); + rsRetVal (*UnreadChar)(strm_t *pThis, uchar c); + rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr); + rsRetVal (*SeekCurrOffs)(strm_t *pThis); + rsRetVal (*Write)(strm_t *pThis, uchar *pBuf, size_t lenBuf); + rsRetVal (*WriteChar)(strm_t *pThis, uchar c); + rsRetVal (*WriteLong)(strm_t *pThis, long i); + rsRetVal (*SetFName)(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); + rsRetVal (*SetDir)(strm_t *pThis, uchar *pszDir, size_t iLenDir); + rsRetVal (*Flush)(strm_t *pThis); + rsRetVal (*RecordBegin)(strm_t *pThis); + rsRetVal (*RecordEnd)(strm_t *pThis); + rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm); + rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); + rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); + rsRetVal (*Dup)(strm_t *pThis, strm_t **ppNew); + INTERFACEpropSetMeth(strm, bDeleteOnClose, int); + INTERFACEpropSetMeth(strm, iMaxFileSize, int); + INTERFACEpropSetMeth(strm, iMaxFiles, int); + INTERFACEpropSetMeth(strm, iFileNumDigits, int); + INTERFACEpropSetMeth(strm, tOperationsMode, int); + INTERFACEpropSetMeth(strm, tOpenMode, mode_t); + INTERFACEpropSetMeth(strm, sType, strmType_t); + INTERFACEpropSetMeth(strm, iZipLevel, int); + INTERFACEpropSetMeth(strm, bSync, int); + INTERFACEpropSetMeth(strm, sIOBufSize, size_t); + INTERFACEpropSetMeth(strm, iSizeLimit, off_t); + INTERFACEpropSetMeth(strm, iFlushInterval, int); + INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); ENDinterface(strm) -#define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ /* prototypes */ -rsRetVal strmConstruct(strm_t **ppThis); -rsRetVal strmConstructFinalize(strm_t __attribute__((unused)) *pThis); -rsRetVal strmDestruct(strm_t **ppThis); -rsRetVal strmSetMaxFileSize(strm_t *pThis, int64 iMaxFileSize); -rsRetVal strmSetFileName(strm_t *pThis, uchar *pszName, size_t iLenName); -rsRetVal strmReadChar(strm_t *pThis, uchar *pC); -rsRetVal strmUnreadChar(strm_t *pThis, uchar c); -rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr); -rsRetVal strmSeekCurrOffs(strm_t *pThis); -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); -rsRetVal strmWriteChar(strm_t *pThis, uchar c); -rsRetVal strmWriteLong(strm_t *pThis, long i); -rsRetVal strmSetFName(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); -rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir); -rsRetVal strmFlush(strm_t *pThis); -rsRetVal strmRecordBegin(strm_t *pThis); -rsRetVal strmRecordEnd(strm_t *pThis); -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm); -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal); -rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs); -rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt); PROTOTYPEObjClassInit(strm); -PROTOTYPEpropSetMeth(strm, bDeleteOnClose, int); -PROTOTYPEpropSetMeth(strm, iMaxFileSize, int); -PROTOTYPEpropSetMeth(strm, iMaxFiles, int); -PROTOTYPEpropSetMeth(strm, iFileNumDigits, int); -PROTOTYPEpropSetMeth(strm, tOperationsMode, int); -PROTOTYPEpropSetMeth(strm, tOpenMode, mode_t); -PROTOTYPEpropSetMeth(strm, sType, strmType_t); #endif /* #ifndef STREAM_H_INCLUDED */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 07256fab..ccf115c1 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -6,8 +6,9 @@ * Please see syslogd.c for license information. * All functions in this "class" start with rsCStr (rsyslog Counted String). * begun 2005-09-07 rgerhards + * did some optimization (read: bugs!) rgerhards, 2009-06-16 * - * Copyright (C) 2007-2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2007-2009 by Rainer Gerhards and Adiscon GmbH * * This file is part of the rsyslog runtime library. * @@ -40,6 +41,7 @@ #include "regexp.h" #include "obj.h" +uchar* rsCStrGetSzStr(cstr_t *pThis); /* ################################################################# * * private members * @@ -54,22 +56,20 @@ DEFobjCurrIf(regexp) * ################################################################# */ -rsRetVal rsCStrConstruct(cstr_t **ppThis) +rsRetVal cstrConstruct(cstr_t **ppThis) { DEFiRet; cstr_t *pThis; ASSERT(ppThis != NULL); - if((pThis = (cstr_t*) calloc(1, sizeof(cstr_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pThis = (cstr_t*) calloc(1, sizeof(cstr_t))); rsSETOBJTYPE(pThis, OIDrsCStr); pThis->pBuf = NULL; pThis->pszBuf = NULL; pThis->iBufSize = 0; pThis->iStrLen = 0; - pThis->iAllocIncrement = RS_STRINGBUF_ALLOC_INCREMENT; *ppThis = pThis; finalize_it: @@ -89,8 +89,8 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); - pThis->iBufSize = pThis->iStrLen = strlen((char*)(char *) sz); - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -119,7 +119,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = pFrom->iStrLen; - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -137,24 +137,8 @@ void rsCStrDestruct(cstr_t **ppThis) { cstr_t *pThis = *ppThis; - /* rgerhards 2005-10-19: The free of pBuf was contained in conditional compilation. - * The code was only compiled if STRINGBUF_TRIM_ALLOCSIZE was set to 1. I honestly - * do not know why it was so, I think it was an artifact. Anyhow, I have changed this - * now. Should there any issue occur, this comment hopefully will shed some light - * on what happened. I re-verified, and this function has never before been called - * by anyone. So changing it can have no impact for obvious reasons... - * - * rgerhards, 2008-02-20: I changed the interface to the new calling conventions, where - * the destructor receives a pointer to the object, so that it can set it to NULL. - */ - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } - - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - } - + free(pThis->pBuf); + free(pThis->pszBuf); RSFREEOBJ(pThis); *ppThis = NULL; } @@ -166,36 +150,32 @@ void rsCStrDestruct(cstr_t **ppThis) * allocated. In practice, a bit more is allocated because we envision that * some more characters may be added after these. * rgerhards, 2008-01-07 + * changed to utilized realloc() -- rgerhards, 2009-06-16 */ -static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) +rsRetVal +rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { - DEFiRet; uchar *pNewBuf; - size_t iNewSize; + unsigned short iNewSize; + DEFiRet; /* first compute the new size needed */ - if(iMinNeeded > pThis->iAllocIncrement) { - /* we allocate "n" iAllocIncrements. Usually, that should + if(iMinNeeded > RS_STRINGBUF_ALLOC_INCREMENT) { + /* we allocate "n" ALLOC_INCREMENTs. Usually, that should * leave some room after the absolutely needed one. It also * reduces memory fragmentation. Note that all of this are * integer operations (very important to understand what is * going on)! Parenthesis are for better readibility. */ - iNewSize = ((iMinNeeded / pThis->iAllocIncrement) + 1) * pThis->iAllocIncrement; + iNewSize = (iMinNeeded / RS_STRINGBUF_ALLOC_INCREMENT + 1) * RS_STRINGBUF_ALLOC_INCREMENT; } else { - iNewSize = pThis->iBufSize + pThis->iAllocIncrement; + iNewSize = pThis->iBufSize + RS_STRINGBUF_ALLOC_INCREMENT; } iNewSize += pThis->iBufSize; /* add current size */ - /* and then allocate and copy over */ /* DEV debugging only: dbgprintf("extending string buffer, old %d, new %d\n", pThis->iBufSize, iNewSize); */ - if((pNewBuf = (uchar*) malloc(iNewSize * sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - memcpy(pNewBuf, pThis->pBuf, pThis->iBufSize); + CHKmalloc(pNewBuf = (uchar*) realloc(pThis->pBuf, iNewSize * sizeof(uchar))); pThis->iBufSize = iNewSize; - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } pThis->pBuf = pNewBuf; finalize_it: @@ -243,7 +223,7 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz) /* append the contents of one cstr_t object to another * rgerhards, 2008-02-25 */ -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) { return rsCStrAppendStrWithLen(pThis, pstrAppend->pBuf, pstrAppend->iStrLen); } @@ -264,45 +244,18 @@ finalize_it: } -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c) -{ - DEFiRet; - - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(pThis->iStrLen >= pThis->iBufSize) { - CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ - } - - /* ok, when we reach this, we have sufficient memory */ - *(pThis->pBuf + pThis->iStrLen++) = c; - - /* check if we need to invalidate an sz representation! */ - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - pThis->pszBuf = NULL; - } - -finalize_it: - RETiRet; -} - - /* Sets the string object to the classigal sz-string provided. * Any previously stored vlaue is discarded. If a NULL pointer * the the new value (pszNew) is provided, an empty string is - * created (this is NOT an error!). Property iAllocIncrement is - * not modified by this function. + * created (this is NOT an error!). * rgerhards, 2005-10-18 */ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->pBuf != NULL) - free(pThis->pBuf); - if(pThis->pszBuf != NULL) - free(pThis->pszBuf); + free(pThis->pBuf); + free(pThis->pszBuf); if(pszNew == NULL) { pThis->iStrLen = 0; pThis->iBufSize = 0; @@ -312,10 +265,9 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) pThis->iStrLen = strlen((char*)pszNew); pThis->iBufSize = pThis->iStrLen; pThis->pszBuf = NULL; - /* iAllocIncrement is NOT modified! */ /* now save the new value */ - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); return RS_RET_OUT_OF_MEMORY; } @@ -363,7 +315,7 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) if(pThis->pBuf != NULL) if(pThis->pszBuf == NULL) { /* we do not yet have a usable sz version - so create it... */ - if((pThis->pszBuf = malloc((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { + if((pThis->pszBuf = MALLOC((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { /* TODO: think about what to do - so far, I have no bright * idea... rgerhards 2005-09-07 */ @@ -395,30 +347,18 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) * MUST be freed by the caller. The function might return NULL if * no memory can be allocated. * - * TODO: - * This function should at some time become special. The base idea is to - * add one extra byte to the end of the regular buffer, so that we can - * convert it to an szString without the need to copy. The extra memory - * footprint is not hefty, but the performance gain is potentially large. - * To get it done now, I am not doing the optimiziation right now. - * rgerhards, 2005-09-07 + * This is the NEW replacement for rsCStrConvSzStrAndDestruct which does + * no longer utilize a special buffer but soley works on pBuf (and also + * assumes that cstrFinalize had been called). * - * rgerhards, 2007-09-04: I have changed the interface of this function. It now - * returns an rsRetVal, so that we can communicate back if we have an error. - * Using the standard method is much better than returning NULL. Secondly, NULL - * was not actually an error - it was in indication if the string was empty. - * This was needed in some parts of the code, in others not. I have now added - * a second parameter to specify what the caller needs. I hope these changes - * will make it less likely that the function is called incorrectly, what - * previously happend quite often and was the cause of a number of program - * aborts. So the parameters are now: + * Parameters are as follows: * pointer to the object, pointer to string-pointer to receive string and * bRetNULL: 0 - must not return NULL on empty string, return "" in that * case, 1 - return NULL instead of an empty string. * PLEASE NOTE: the caller must free the memory returned in ppSz in any case * (except, of course, if it is NULL). */ -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) { DEFiRet; uchar* pRetBuf; @@ -429,14 +369,13 @@ rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - if((pRetBuf = malloc(sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pRetBuf = MALLOC(sizeof(uchar))); *pRetBuf = '\0'; } else { pRetBuf = NULL; } } else - pRetBuf = rsCStrGetSzStr(pThis); + pRetBuf = pThis->pBuf; *ppSz = pRetBuf; @@ -445,64 +384,11 @@ finalize_it: * that we can NOT use the rsCStrDestruct function as it would * also free the sz String buffer, which we pass on to the user. */ - if(pThis->pBuf != NULL) - free(pThis->pBuf); RSFREEOBJ(pThis); - RETiRet; } -#if STRINGBUF_TRIM_ALLOCSIZE == 1 - /* Only in this mode, we need to trim the string. To do - * so, we must allocate a new buffer of the exact - * string size, and then copy the old one over. - */ - /* WARNING - * STRINGBUF_TRIM_ALLOCSIZE can, in theory, be used to trim - * memory buffers. This part of the code was inherited from - * liblogging (where it is used in a different context) but - * never put to use in rsyslog. The reason is that it is hardly - * imaginable where the extra performance cost is worth the save - * in memory alloc. Then Anders Blomdel rightfully pointed out that - * the code does not work at all - and nobody even know that it - * probably shouldn't. Rather than removing, I deciced to somewhat - * fix the code, so that this feature may be enabled if somebody - * really has a need for it. Be warned, however, that I NEVER - * tested the fix. So if you intend to use this feature, you must - * do full testing before you rely on it. -- rgerhards, 2008-02-12 - */ -rsRetVal rsCStrFinish(cstr_t __attribute__((unused)) *pThis) -{ - DEFiRet; - uchar* pBuf; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if((pBuf = malloc((pThis->iStrLen) * sizeof(uchar))) == NULL) - { /* OK, in this case we use the previous buffer. At least - * we have it ;) - */ - } - else - { /* got the new buffer, so let's use it */ - memcpy(pBuf, pThis->pBuf, pThis->iStrLen); - pThis->pBuf = pBuf; - } - - RETiRet; -} -#endif /* #if STRINGBUF_TRIM_ALLOCSIZE == 1 */ - - -void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) -{ - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - assert(iNewIncrement > 0); - - pThis->iAllocIncrement = iNewIncrement; -} - - /* return the length of the current string * 2005-09-09 rgerhards * Please note: this is only a function in a debug build. @@ -510,7 +396,7 @@ void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) * This is due to performance reasons. */ #ifndef NDEBUG -int rsCStrLen(cstr_t *pThis) +int cstrLen(cstr_t *pThis) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); return(pThis->iStrLen); @@ -561,6 +447,27 @@ rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis) return RS_RET_OK; } +/* Trim trailing whitespace from a given string + */ +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis) +{ + register int i; + register uchar *pC; + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + i = pThis->iStrLen; + pC = pThis->pBuf + i - 1; + while(i > 0 && isspace((int)*pC)) { + --pC; + --i; + } + /* i now is the new string length! */ + pThis->iStrLen = i; + pThis->pBuf[pThis->iStrLen] = '0'; /* we always have this space */ + + return RS_RET_OK; +} + /* compare two string objects - works like strcmp(), but operates * on CStr objects. Please note that this version here is * faster in the majority of cases, simply because it can @@ -1027,56 +934,6 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz) } -#if 0 /* read comment below why this is commented out. In short: for future use! */ -/* locate the first occurence of a standard sz string inside a rsCStr object. - * Returns the offset (0-bound) of this first occurrence. If not found, -1 is - * returned. - * rgerhards 2005-09-19 - * WARNING: I accidently created this function (I later noticed I didn't relly - * need it... I will not remove the function, as it probably is useful - * some time later. However, it is not fully tested, so start with testing - * it before you put it to first use). - */ -int rsCStrLocateSzStr(cstr_t *pThis, uchar *sz) -{ - int iLenSz; - int i; - int iMax; - int bFound; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(sz == NULL) - return 0; - - iLenSz = strlen((char*)sz); - if(iLenSz == 0) - return 0; - - /* compute the largest index where a match could occur - after all, - * the to-be-located string must be able to be present in the - * searched string (it needs its size ;)). - */ - iMax = pThis->iStrLen - iLenSz; - - bFound = 0; - i = 0; - while(i < iMax && !bFound) { - int iCheck; - uchar *pComp = pThis->pBuf + i; - for(iCheck = 0 ; iCheck < iLenSz ; ++iCheck) - if(*(pComp + iCheck) != *(sz + iCheck)) - break; - if(iCheck == iLenSz) - bFound = 1; /* found! - else it wouldn't be equal */ - else - ++i; /* on to the next try */ - } - - return(bFound ? i : -1); -} -#endif /* end comment out */ - - /* our exit function. TODO: remove once converted to a class * rgerhards, 2008-03-11 */ @@ -1100,11 +957,5 @@ finalize_it: } -/* - * Local variables: - * c-indent-level: 8 - * c-basic-offset: 8 - * tab-width: 8 - * End: - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 684133bb..c5130238 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -1,5 +1,5 @@ -/*! \file stringbuf.h - * \brief The counted string object +/* stringbuf.h + * The counted string object * * This is the byte-counted string class for rsyslog. It is a replacement * for classical \0 terminated string functions. We introduce it in @@ -11,8 +11,7 @@ * \date 2005-09-07 * Initial version begun. * - * All functions in this "class" start with rsCStr (rsyslog Counted String). - * Copyright 2005 + * Copyright 2005-2009 * Rainer Gerhards and Adiscon GmbH. All Rights Reserved. * * This file is part of the rsyslog runtime library. @@ -36,6 +35,8 @@ #ifndef _STRINGBUF_H_INCLUDED__ #define _STRINGBUF_H_INCLUDED__ 1 +#include <assert.h> + /** * The dynamic string buffer object. */ @@ -48,14 +49,14 @@ typedef struct cstr_s uchar *pszBuf; /**< pointer to the sz version of the string (after it has been created )*/ size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ - size_t iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ } cstr_t; /** * Construct a rsCStr object. */ -rsRetVal rsCStrConstruct(cstr_t **ppThis); +rsRetVal cstrConstruct(cstr_t **ppThis); +#define rsCStrConstruct(x) cstrConstruct((x)) rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz); rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); @@ -63,14 +64,88 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); * Destruct the string buffer object. */ void rsCStrDestruct(cstr_t **ppThis); +#define cstrDestruct(x) rsCStrDestruct((x)) -/** - * Append a character to an existing string. If necessary, the - * method expands the string buffer. - * - * \param c Character to append to string. + +/* Append a character to the current string object. This may only be done until + * cstrFinalize() is called. + * rgerhards, 2009-06-16 + */ +rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded); /* our helper, NOT a public interface! */ +static inline rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) +{ + rsRetVal iRet = RS_RET_OK; + + if(pThis->iStrLen >= pThis->iBufSize) { + CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ + } + + /* ok, when we reach this, we have sufficient memory */ + *(pThis->pBuf + pThis->iStrLen++) = c; + +finalize_it: + return iRet; +} + + +/* some inline functions for things that are really frequently called... */ + +/* Finalize the string object. This must be called after all data is added to it + * but before that data is used. + * rgerhards, 2009-06-16 + */ +static inline rsRetVal +cstrFinalize(cstr_t *pThis) +{ + rsRetVal iRet = RS_RET_OK; + + if(pThis->iStrLen > 0) { + /* terminate string only if one exists */ + CHKiRet(cstrAppendChar(pThis, '\0')); + --pThis->iStrLen; /* do NOT count the \0 byte */ + } + +finalize_it: + return iRet; +} + + +/* Returns the cstr data as a classical C sz string. We use that the + * Finalizer did properly terminate our string (but we may stil be NULL). + * So it is vital that the finalizer is called BEFORe this function here! + * The caller must not free or otherwise manipulate the returned string and must not + * destroy the CStr object as long as the ascii string is used. + * This function may return NULL, if the string is currently NULL. This + * is a feature, not a bug. If you need non-NULL in any case, use + * cstrGetSzStrNoNULL() instead. + * Note that due to the new single-buffer interface this function almost does nothing! + * rgerhards, 2006-09-16 */ -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); +static inline uchar* cstrGetSzStr(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + return(pThis->pBuf); +} + + +/* Converts the CStr object to a classical sz string and returns that. + * Same restrictions as in cstrGetSzStr() applies (see there!). This + * function here guarantees that a valid string is returned, even if + * the CStr object currently holds a NULL pointer string buffer. If so, + * "" is returned. + * rgerhards 2005-10-19 + * WARNING: The returned pointer MUST NOT be freed, as it may be + * obtained from that constant memory pool (in case of NULL!) + */ +static inline uchar* cstrGetSzStrNoNULL(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + if(pThis->pBuf == NULL) + return (uchar*) ""; + else + return cstrGetSzStr(pThis); +} + /** * Truncate "n" number of characters from the end of the @@ -81,6 +156,7 @@ rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); rsRetVal rsCStrTruncate(cstr_t *pThis, size_t nTrunc); rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis); +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis); /** * Append a string to the buffer. For performance reasons, @@ -98,22 +174,6 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz); */ rsRetVal rsCStrAppendStrWithLen(cstr_t *pThis, uchar* psz, size_t iStrLen); -/** - * Set a new allocation incremet. This will influence - * the allocation the next time the string will be expanded. - * It can be set and changed at any time. If done immediately - * after custructing the StrB object, this will also be - * the inital allocation. - * - * \param iNewIncrement The new increment size - * - * \note It is possible to use a very low increment, e.g. 1 byte. - * This can generate a considerable overhead. We highly - * advise not to use an increment below 32 bytes, except - * if you are very well aware why you are doing it ;) - */ -void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement); -#define rsCStrGetAllocIncrement(pThis) ((pThis)->iAllocIncrement) /** * Append an integer to the string. No special formatting is @@ -123,10 +183,9 @@ rsRetVal rsCStrAppendInt(cstr_t *pThis, long i); rsRetVal strExit(void); /* TODO: remove once we have a real object interface! */ -uchar* rsCStrGetSzStr(cstr_t *pThis); +uchar* __attribute__((deprecated)) rsCStrGetSzStr(cstr_t *pThis); uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis); rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew); -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); int rsCStrCStrCmp(cstr_t *pCS1, cstr_t *pCS2); int rsCStrSzStrCmp(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrOffsetSzStrCmp(cstr_t *pCS1, size_t iOffset, uchar *psz, size_t iLenSz); @@ -140,27 +199,22 @@ rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *cache) void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); + +/* in migration */ +#define rsCStrAppendCStr(pThis, pstrAppend) cstrAppendCStr(pThis, pstrAppend) + +/* new calling interface */ +rsRetVal cstrFinalize(cstr_t *pThis); +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); /* now come inline-like functions */ #ifdef NDEBUG -# define rsCStrLen(x) ((int)((x)->iStrLen)) -#else - int rsCStrLen(cstr_t *pThis); -#endif - -#if STRINGBUF_TRIM_ALLOCSIZE != 1 -/* This is the normal case (see comment in rsCStrFinish!). In those cases, the function - * simply needs to do nothing, so that we can save us the function call. - * rgerhards, 2008-02-12 - */ -# define rsCStrFinish(pThis) RS_RET_OK +# define cstrLen(x) ((int)((x)->iStrLen)) #else - /** - * Finish the string buffer dynamic allocation. - */ - rsRetVal rsCStrFinish(cstr_t *pThis); + int cstrLen(cstr_t *pThis); #endif +#define rsCStrLen(s) cstrLen((s)) #define rsCStrGetBufBeg(x) ((x)->pBuf) diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c new file mode 100644 index 00000000..0aeebe03 --- /dev/null +++ b/runtime/strms_sess.c @@ -0,0 +1,300 @@ +/* strms_sess.c + * + * This implements a session of the strmsrv object. For general + * comments, see header of strmsrv.c. + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include <ctype.h> + +#include "rsyslog.h" +#include "dirty.h" +#include "module-template.h" +#include "net.h" +#include "strmsrv.h" +#include "strms_sess.h" +#include "obj.h" +#include "errmsg.h" +#include "netstrm.h" +#include "msg.h" +#include "datetime.h" + + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(datetime) + +static int iMaxLine; /* maximum size of a single message */ + +/* forward definitions */ +static rsRetVal Close(strms_sess_t *pThis); + + +/* Standard-Constructor */ +BEGINobjConstruct(strms_sess) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(strms_sess) + + +/* ConstructionFinalizer + */ +static rsRetVal +strms_sessConstructFinalize(strms_sess_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + if(pThis->pSrv->OnSessConstructFinalize != NULL) { + CHKiRet(pThis->pSrv->OnSessConstructFinalize(&pThis->pUsr)); + } + +finalize_it: + RETiRet; +} + + +/* destructor for the strms_sess object */ +BEGINobjDestruct(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strms_sess) + if(pThis->pStrm != NULL) + netstrm.Destruct(&pThis->pStrm); + + if(pThis->pSrv->pOnSessDestruct != NULL) { + pThis->pSrv->pOnSessDestruct(&pThis->pUsr); + } + /* now destruct our own properties */ + free(pThis->fromHost); + free(pThis->fromHostIP); +ENDobjDestruct(strms_sess) + + +/* debugprint for the strms_sess object */ +BEGINobjDebugPrint(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strms_sess) +ENDobjDebugPrint(strms_sess) + + +/* set property functions */ +/* set the hostname. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHost() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-04-24 + */ +static rsRetVal +SetHost(strms_sess_t *pThis, uchar *pszHost) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHost); + pThis->fromHost = pszHost; + RETiRet; +} + +/* set the remote host's IP. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHostIP() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-05-16 + */ +static rsRetVal +SetHostIP(strms_sess_t *pThis, uchar *pszHostIP) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHostIP); + pThis->fromHostIP = pszHostIP; + RETiRet; +} + +static rsRetVal +SetStrm(strms_sess_t *pThis, netstrm_t *pStrm) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + pThis->pStrm = pStrm; + RETiRet; +} + + +/* set our parent, the strmsrv object */ +static rsRetVal +SetStrmsrv(strms_sess_t *pThis, strmsrv_t *pSrv) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + ISOBJ_TYPE_assert(pSrv, strmsrv); + pThis->pSrv = pSrv; + RETiRet; +} + + +/* set our parent listener info*/ +static rsRetVal +SetLstnInfo(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pLstnInfo != NULL); + pThis->pLstnInfo = pLstnInfo; + RETiRet; +} + + +static rsRetVal +SetUsrP(strms_sess_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + + +static void * +GetUsrP(strms_sess_t *pThis) +{ + return pThis->pUsr; +} + + +/* Closes a STRM session + * No attention is paid to the return code + * of close, so potential-double closes are not detected. + */ +static rsRetVal +Close(strms_sess_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strms_sess); + netstrm.Destruct(&pThis->pStrm); + free(pThis->fromHost); + pThis->fromHost = NULL; /* not really needed, but... */ + free(pThis->fromHostIP); + pThis->fromHostIP = NULL; /* not really needed, but... */ + + RETiRet; +} + + + +/* Processes the data received via a STRM session. If there + * is no other way to handle it, data is discarded. + * Input parameter data is the data received, iLen is its + * len as returned from recv(). iLen must be 1 or more (that + * is errors must be handled by caller!). iSTRMSess must be + * the index of the STRM session that received the data. + * rgerhards 2005-07-04 + * And another change while generalizing. We now return either + * RS_RET_OK, which means the session should be kept open + * or anything else, which means it must be closed. + * rgerhards, 2008-03-01 + */ +static rsRetVal +DataRcvd(strms_sess_t *pThis, char *pData, size_t iLen) +{ + DEFiRet; + char *pEnd; + + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pData != NULL); + assert(iLen > 0); + + /* We now copy the message to the session buffer. */ + pEnd = pData + iLen; /* this is one off, which is intensional */ + + while(pData < pEnd) { + CHKiRet(pThis->pSrv->OnCharRcvd(pThis, (uchar)*pData++)); + } + +finalize_it: + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strms_sess) +CODESTARTobjQueryInterface(strms_sess) + if(pIf->ifVersion != strms_sessCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strms_sessDebugPrint; + pIf->Construct = strms_sessConstruct; + pIf->ConstructFinalize = strms_sessConstructFinalize; + pIf->Destruct = strms_sessDestruct; + + pIf->Close = Close; + pIf->DataRcvd = DataRcvd; + + pIf->SetUsrP = SetUsrP; + pIf->GetUsrP = GetUsrP; + pIf->SetStrmsrv = SetStrmsrv; + pIf->SetLstnInfo = SetLstnInfo; + pIf->SetHost = SetHost; + pIf->SetHostIP = SetHostIP; + pIf->SetStrm = SetStrm; +finalize_it: +ENDobjQueryInterface(strms_sess) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strms_sess, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strms_sess) + /* release objects we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(datetime, CORE_COMPONENT); +ENDObjClassExit(strms_sess) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strms_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + CHKiRet(objUse(glbl, CORE_COMPONENT)); + iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ + objRelease(glbl, CORE_COMPONENT); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strms_sessDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strms_sessConstructFinalize); +ENDObjClassInit(strms_sess) + +/* vim:set ai: + */ diff --git a/runtime/strms_sess.h b/runtime/strms_sess.h new file mode 100644 index 00000000..6483c0c3 --- /dev/null +++ b/runtime/strms_sess.h @@ -0,0 +1,75 @@ +/* Definitions for strms_sess class. This implements a session of the + * generic stream server. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMS_SESS_H +#define INCLUDED_STRMS_SESS_H + +#include "obj.h" + +/* a forward-definition, we are somewhat cyclic */ +struct strmsrv_s; + +/* the strms_sess object */ +struct strms_sess_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + strmsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ + strmLstnPortList_t *pLstnInfo; /* pointer back to listener info */ + netstrm_t *pStrm; +// uchar *pMsg; /* message (fragment) received */ + uchar *fromHost; + uchar *fromHostIP; + void *pUsr; /* a user-pointer */ +}; + + +/* interfaces */ +BEGINinterface(strms_sess) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strms_sess); + rsRetVal (*Construct)(strms_sess_t **ppThis); + rsRetVal (*ConstructFinalize)(strms_sess_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strms_sess_t **ppThis); + rsRetVal (*Close)(strms_sess_t *pThis); + rsRetVal (*DataRcvd)(strms_sess_t *pThis, char *pData, size_t iLen); + /* set methods */ + rsRetVal (*SetStrmsrv)(strms_sess_t *pThis, struct strmsrv_s *pSrv); + rsRetVal (*SetLstnInfo)(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo); + rsRetVal (*SetUsrP)(strms_sess_t*, void*); + void* (*GetUsrP)(strms_sess_t*); + rsRetVal (*SetHost)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetHostIP)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetStrm)(strms_sess_t *pThis, netstrm_t*); + rsRetVal (*SetOnMsgReceive)(strms_sess_t *pThis, rsRetVal (*OnMsgReceive)(strms_sess_t*, uchar*, int)); +ENDinterface(strms_sess) +#define strms_sessCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface changes + * to version v2, rgerhards, 2009-05-22 + * - Data structures changed + * - SetLstnInfo entry point added + */ + + +/* prototypes */ +PROTOTYPEObj(strms_sess); + + +#endif /* #ifndef INCLUDED_STRMS_SESS_H */ diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c new file mode 100644 index 00000000..a122ca8a --- /dev/null +++ b/runtime/strmsrv.c @@ -0,0 +1,966 @@ +/* strmsrv.c + * + * This builds a basic stream server. It handles connection creation but + * not any protocol. Instead, it calls a "data received" entry point of the + * caller with any data received, in which case the caller must react accordingly. + * This module works together with the netstream drivers. + * + * There are actually two classes within the stream server code: one is + * the strmsrv itself, the other one is its sessions. This is a helper + * class to strmsrv. + * + * File begun on 2009-06-01 by RGerhards based on strmsrv.c. Note that strmsrv is + * placed under LGPL, which is possible because I carefully evaluated and + * eliminated all those parts of strmsrv which were not written by me. + * + * TODO: I would consider it useful to migrate tcpsrv.c/tcps_sess.c to this stream + * class here. The requires a little bit redesign, but should not be too hard. The + * core idea, already begun here, is that we still support lots of callbacks, but + * provide "canned" implementations for standard cases. That way, most upper-layer + * modules can be kept rather simple and without any extra overhead. Note that + * to support this, tcps_sess.c would need to extract the message reception state + * machine to a separate module which then is called via the DoCharRcvd() interface + * of this class here. -- rgerhards, 2009-06-01 + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <assert.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <stdarg.h> +#include <ctype.h> +#include <netinet/in.h> +#include <netdb.h> +#include <sys/types.h> +#include <sys/socket.h> +#if HAVE_FCNTL_H +#include <fcntl.h> +#endif +#include "rsyslog.h" +#include "dirty.h" +#include "cfsysline.h" +#include "module-template.h" +#include "net.h" +#include "srUtils.h" +#include "conf.h" +#include "strmsrv.h" +#include "obj.h" +#include "glbl.h" +#include "netstrms.h" +#include "netstrm.h" +#include "nssel.h" +#include "errmsg.h" +#include "unicode-helper.h" + +MODULE_TYPE_LIB + +/* defines */ +#define STRMSESS_MAX_DEFAULT 200 /* default for nbr of strm sessions if no number is given */ +#define STRMLSTN_MAX_DEFAULT 20 /* default for nbr of listeners */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(conf) +DEFobjCurrIf(glbl) +DEFobjCurrIf(strms_sess) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(net) +DEFobjCurrIf(netstrms) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(nssel) + +/* forward definitions */ +static rsRetVal create_strm_socket(strmsrv_t *pThis); + +/* standard callbacks, if the caller did not provide us with them (this helps keep us + * flexible while at the same time permits very simple upper-layer modules) + */ +/* this shall go into a specific ACL module! */ +static int +isPermittedHost(struct sockaddr __attribute__((unused)) *addr, char __attribute__((unused)) *fromHostFQDN, + void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) +{ + return 1; +} + + +static rsRetVal +doOpenLstnSocks(strmsrv_t *pSrv) +{ + ISOBJ_TYPE_assert(pSrv, strmsrv); + return create_strm_socket(pSrv); +} + + +static rsRetVal +doRcvData(strms_sess_t *pSess, char *buf, size_t lenBuf, ssize_t *piLenRcvd) +{ + DEFiRet; + assert(pSess != NULL); + assert(piLenRcvd != NULL); + + *piLenRcvd = lenBuf; + CHKiRet(netstrm.Rcv(pSess->pStrm, (uchar*) buf, piLenRcvd)); +finalize_it: + RETiRet; +} + +static rsRetVal +onRegularClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + /* process any incomplete frames left over */ + //strms_sess.PrepareClose(pSess); + /* Session closed */ + strms_sess.Close(pSess); + RETiRet; +} + + +static rsRetVal +onErrClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + strms_sess.Close(pSess); + RETiRet; +} + +/* ------------------------------ end callbacks ------------------------------ */ + +/* add new listener port to listener port list + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +addNewLstnPort(strmsrv_t *pThis, uchar *pszPort) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* create entry */ + CHKmalloc(pEntry = MALLOC(sizeof(strmLstnPortList_t))); + pEntry->pszPort = pszPort; + pEntry->pSrv = pThis; + CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + + /* and add to list */ + pEntry->pNext = pThis->pLstnPorts; + pThis->pLstnPorts = pEntry; + +finalize_it: + RETiRet; +} + + +/* configure STRM listener settings. + * Note: pszPort is handed over to us - the caller MUST NOT free it! + * rgerhards, 2008-03-20 + */ +static rsRetVal +configureSTRMListen(strmsrv_t *pThis, uchar *pszPort) +{ + int i; + uchar *pPort = pszPort; + DEFiRet; + + assert(pszPort != NULL); + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* extract port */ + i = 0; + while(isdigit((int) *pPort)) { + i = i * 10 + *pPort++ - '0'; + } + + if(i >= 0 && i <= 65535) { + CHKiRet(addNewLstnPort(pThis, pszPort)); + } else { + errmsg.LogError(0, NO_ERRCODE, "Invalid STRM listen port %s - ignored.\n", pszPort); + } + +finalize_it: + RETiRet; +} + + +/* Initialize the session table + * returns 0 if OK, somewhat else otherwise + */ +static rsRetVal +STRMSessTblInit(strmsrv_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions == NULL); + + dbgprintf("Allocating buffer for %d STRM sessions.\n", pThis->iSessMax); + if((pThis->pSessions = (strms_sess_t **) calloc(pThis->iSessMax, sizeof(strms_sess_t *))) == NULL) { + dbgprintf("Error: STRMSessInit() could not alloc memory for STRM session table.\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + +finalize_it: + RETiRet; +} + + +/* find a free spot in the session table. If the table + * is full, -1 is returned, else the index of the free + * entry (0 or higher). + */ +static int +STRMSessTblFindFreeSpot(strmsrv_t *pThis) +{ + register int i; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + for(i = 0 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] == NULL) + break; + } + + return((i < pThis->iSessMax) ? i : -1); +} + + +/* Get the next session index. Free session tables entries are + * skipped. This function is provided the index of the last + * session entry, or -1 if no previous entry was obtained. It + * returns the index of the next session or -1, if there is no + * further entry in the table. Please note that the initial call + * might as well return -1, if there is no session at all in the + * session table. + */ +static int +STRMSessGetNxtSess(strmsrv_t *pThis, int iCurr) +{ + register int i; + + BEGINfunc + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions != NULL); + for(i = iCurr + 1 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] != NULL) + break; + } + + ENDfunc + return((i < pThis->iSessMax) ? i : -1); +} + + +/* De-Initialize STRM listner sockets. + * This function deinitializes everything, including freeing the + * session table. No STRM listen receive operations are permitted + * unless the subsystem is reinitialized. + * rgerhards, 2007-06-21 + */ +static void deinit_strm_listener(strmsrv_t *pThis) +{ + int i; + strmLstnPortList_t *pEntry; + strmLstnPortList_t *pDel; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + if(pThis->pSessions != NULL) { + /* close all STRM connections! */ + i = STRMSessGetNxtSess(pThis, -1); + while(i != -1) { + strms_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = STRMSessGetNxtSess(pThis, i); + } + + /* we are done with the session table - so get rid of it... */ + free(pThis->pSessions); + pThis->pSessions = NULL; /* just to make sure... */ + } + + /* free list of strm listen ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + free(pEntry->pszPort); + free(pEntry->pszInputName); + pDel = pEntry; + pEntry = pEntry->pNext; + free(pDel); + } + + /* finally close our listen streams */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + netstrm.Destruct(pThis->ppLstn + i); + } +} + + +/* add a listen socket to our listen socket array. This is a callback + * invoked from the netstrm class. -- rgerhards, 2008-04-23 + */ +static rsRetVal +addStrmLstn(void *pUsr, netstrm_t *pLstn) +{ + strmLstnPortList_t *pPortList = (strmLstnPortList_t *) pUsr; + strmsrv_t *pThis = pPortList->pSrv; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + ISOBJ_TYPE_assert(pLstn, netstrm); + + if(pThis->iLstnMax >= STRMLSTN_MAX_DEFAULT) + ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); + + pThis->ppLstn[pThis->iLstnMax] = pLstn; + pThis->ppLstnPort[pThis->iLstnMax] = pPortList; + ++pThis->iLstnMax; + +finalize_it: + RETiRet; +} + + +/* Initialize STRM listener socket for a single port + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +initSTRMListener(strmsrv_t *pThis, strmLstnPortList_t *pPortEntry) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pPortEntry != NULL); + + /* TODO: add capability to specify local listen address! */ + CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pPortEntry, addStrmLstn, pPortEntry->pszPort, NULL, pThis->iSessMax)); + +finalize_it: + RETiRet; +} + + +/* Initialize STRM sockets (for listener) and listens on them */ +static rsRetVal +create_strm_socket(strmsrv_t *pThis) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* init all configured ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + CHKiRet(initSTRMListener(pThis, pEntry)); + pEntry = pEntry->pNext; + } + + /* OK, we had success. Now it is also time to + * initialize our connections + */ + if(STRMSessTblInit(pThis) != 0) { + /* OK, we are in some trouble - we could not initialize the + * session table, so we can not continue. We need to free all + * we have assigned so far, because we can not really use it... + */ + errmsg.LogError(0, RS_RET_ERR, "Could not initialize STRM session table, suspending STRM message reception."); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Accept new STRM connection; make entry in session table. If there + * is no more space left in the connection table, the new STRM + * connection is immediately dropped. + * ppSess has a pointer to the newly created session, if it succeeds. + * If it does not succeed, no session is created and ppSess is + * undefined. If the user has provided an OnSessAccept Callback, + * this one is executed immediately after creation of the + * session object, so that it can do its own initialization. + * rgerhards, 2008-03-02 + */ +static rsRetVal +SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSess, netstrm_t *pStrm) +{ + DEFiRet; + strms_sess_t *pSess = NULL; + netstrm_t *pNewStrm = NULL; + int iSess = -1; + struct sockaddr_storage *addr; + uchar *fromHostFQDN = NULL; + uchar *fromHostIP = NULL; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pLstnInfo != NULL); + + CHKiRet(netstrm.AcceptConnReq(pStrm, &pNewStrm)); + + /* Add to session list */ + iSess = STRMSessTblFindFreeSpot(pThis); + if(iSess == -1) { + errno = 0; + errmsg.LogError(0, RS_RET_MAX_SESS_REACHED, "too many strm sessions - dropping incoming request"); + ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); + } + + if(pThis->bUseKeepAlive) { + CHKiRet(netstrm.EnableKeepAlive(pNewStrm)); + } + + /* we found a free spot and can construct our session object */ + CHKiRet(strms_sess.Construct(&pSess)); + CHKiRet(strms_sess.SetStrmsrv(pSess, pThis)); + CHKiRet(strms_sess.SetLstnInfo(pSess, pLstnInfo)); + + /* get the host name */ + CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); + CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); + CHKiRet(netstrm.GetRemAddr(pNewStrm, &addr)); + /* TODO: check if we need to strip the domain name here -- rgerhards, 2008-04-24 */ + + /* Here we check if a host is permitted to send us messages. If it isn't, we do not further + * process the message but log a warning (if we are configured to do this). + * rgerhards, 2005-09-26 + */ + if(pThis->pIsPermittedHost != NULL + && !pThis->pIsPermittedHost((struct sockaddr*) addr, (char*) fromHostFQDN, pThis->pUsr, pSess->pUsr)) { + dbgprintf("%s is not an allowed sender\n", fromHostFQDN); + if(glbl.GetOption_DisallowWarning()) { + errno = 0; + errmsg.LogError(0, RS_RET_HOST_NOT_PERMITTED, "STRM message from disallowed sender %s discarded", fromHostFQDN); + } + ABORT_FINALIZE(RS_RET_HOST_NOT_PERMITTED); + } + + /* OK, we have an allowed sender, so let's continue, what + * means we can finally fill in the session object. + */ + CHKiRet(strms_sess.SetHost(pSess, fromHostFQDN)); + fromHostFQDN = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetHostIP(pSess, fromHostIP)); + fromHostIP = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetStrm(pSess, pNewStrm)); + pNewStrm = NULL; /* prevent it from being freed in error handler, now done in strms_sess! */ + CHKiRet(strms_sess.ConstructFinalize(pSess)); + + /* check if we need to call our callback */ + if(pThis->pOnSessAccept != NULL) { + CHKiRet(pThis->pOnSessAccept(pThis, pSess)); + } + + *ppSess = pSess; + pThis->pSessions[iSess] = pSess; + pSess = NULL; /* this is now also handed over */ + +finalize_it: + if(iRet != RS_RET_OK) { + if(pSess != NULL) + strms_sess.Destruct(&pSess); + if(pNewStrm != NULL) + netstrm.Destruct(&pNewStrm); + free(fromHostFQDN); + free(fromHostIP); + } + + RETiRet; +} + + +static void +RunCancelCleanup(void *arg) +{ + nssel_t **ppSel = (nssel_t**) arg; + + if(*ppSel != NULL) + nssel.Destruct(ppSel); +} + + +/* This function is called to gather input. */ +#pragma GCC diagnostic ignored "-Wempty-body" +static rsRetVal +Run(strmsrv_t *pThis) +{ + DEFiRet; + int nfds; + int i; + int iSTRMSess; + int bIsReady; + strms_sess_t *pNewSess; + nssel_t *pSel; + ssize_t iRcvd; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerharsd, 20080-04-24 + */ + pthread_cleanup_push(RunCancelCleanup, (void*) &pSel); + while(1) { + CHKiRet(nssel.Construct(&pSel)); + // TODO: set driver + CHKiRet(nssel.ConstructFinalize(pSel)); + + /* Add the STRM listen sockets to the list of read descriptors. */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.Add(pSel, pThis->ppLstn[i], NSDSEL_RD)); + } + + /* do the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(iSTRMSess != -1) { + /* TODO: access to pNsd is NOT really CLEAN, use method... */ + CHKiRet(nssel.Add(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD)); + /* now get next... */ + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + + /* wait for io to become ready */ + CHKiRet(nssel.Wait(pSel, &nfds)); + + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + --nfds; /* indicate we have processed one */ + } + } + + /* now check the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(nfds && iSTRMSess != -1) { + CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + dbgprintf("netstream %p with new data\n", pThis->pSessions[iSTRMSess]->pStrm); + + /* Receive message */ + iRet = pThis->pRcvData(pThis->pSessions[iSTRMSess], buf, sizeof(buf), &iRcvd); + switch(iRet) { + case RS_RET_CLOSED: + pThis->pOnRegularClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + case RS_RET_RETRY: + /* we simply ignore retry - this is not an error, but we also have not received anything */ + break; + case RS_RET_OK: + /* valid data received, process it! */ + if(strms_sess.DataRcvd(pThis->pSessions[iSTRMSess], buf, iRcvd) != RS_RET_OK) { + /* in this case, something went awfully wrong. + * We are instructed to terminate the session. + */ + errmsg.LogError(0, NO_ERRCODE, "Tearing down STRM Session %d - see " + "previous messages for reason(s)\n", iSTRMSess); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + } + break; + default: + errno = 0; + errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", + pThis->pSessions[iSTRMSess]->pStrm); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + } + --nfds; /* indicate we have processed one */ + } + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + CHKiRet(nssel.Destruct(&pSel)); +finalize_it: /* this is a very special case - this time only we do not exit the function, + * because that would not help us either. So we simply retry it. Let's see + * if that actually is a better idea. Exiting the loop wasn't we always + * crashed, which made sense (the rest of the engine was not prepared for + * that) -- rgerhards, 2008-05-19 + */ + /*EMPTY*/; + } + + /* note that this point is usually not reached */ + pthread_cleanup_pop(0); /* remove cleanup handler */ + + RETiRet; +} +#pragma GCC diagnostic warning "-Wempty-body" + + +/* Standard-Constructor */ +BEGINobjConstruct(strmsrv) /* be sure to specify the object type also in END macro! */ + pThis->iSessMax = STRMSESS_MAX_DEFAULT; /* TODO: useful default ;) */ + /* set default callbacks (used if caller does not overwrite them) */ + pThis->pIsPermittedHost = isPermittedHost; + pThis->OpenLstnSocks = doOpenLstnSocks; + pThis->pRcvData = doRcvData; + pThis->pOnRegularClose = onRegularClose; + pThis->pOnErrClose = onErrClose; + /* session specific callbacks */ + //pThis->OnSessConstructFinalize = + //pThis->pOnSessDestruct = +ENDobjConstruct(strmsrv) + + +/* ConstructionFinalizer */ +static rsRetVal +strmsrvConstructFinalize(strmsrv_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* prepare network stream subsystem */ + CHKiRet(netstrms.Construct(&pThis->pNS)); + CHKiRet(netstrms.SetDrvrMode(pThis->pNS, pThis->iDrvrMode)); + if(pThis->pszDrvrAuthMode != NULL) + CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); + if(pThis->pPermPeers != NULL) + CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); + // TODO: set driver! + CHKiRet(netstrms.ConstructFinalize(pThis->pNS)); + + /* set up listeners */ + CHKmalloc(pThis->ppLstn = calloc(STRMLSTN_MAX_DEFAULT, sizeof(netstrm_t*))); + CHKmalloc(pThis->ppLstnPort = calloc(STRMLSTN_MAX_DEFAULT, sizeof(strmLstnPortList_t*))); + iRet = pThis->OpenLstnSocks(pThis); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + } + RETiRet; +} + + +/* destructor for the strmsrv object */ +BEGINobjDestruct(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strmsrv) + if(pThis->OnDestruct != NULL) + pThis->OnDestruct(pThis->pUsr); + + deinit_strm_listener(pThis); + + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + free(pThis->pszDrvrAuthMode); + free(pThis->ppLstn); + free(pThis->ppLstnPort); + free(pThis->pszInputName); +ENDobjDestruct(strmsrv) + + +/* debugprint for the strmsrv object */ +BEGINobjDebugPrint(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strmsrv) +ENDobjDebugPrint(strmsrv) + +/* set functions */ +static rsRetVal +SetCBIsPermittedHost(strmsrv_t *pThis, int (*pCB)(struct sockaddr *addr, char *fromHostFQDN, void*, void*)) +{ + DEFiRet; + pThis->pIsPermittedHost = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessAccept(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*, strms_sess_t*)) +{ + DEFiRet; + pThis->pOnSessAccept = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessConstructFinalize(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnSessConstructFinalize = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->pOnSessDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnRegularClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnRegularClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnErrClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnErrClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOpenLstnSocks(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*)) +{ + DEFiRet; + pThis->OpenLstnSocks = pCB; + RETiRet; +} + +static rsRetVal +SetUsrP(strmsrv_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + +static rsRetVal +SetKeepAlive(strmsrv_t *pThis, int iVal) +{ + DEFiRet; + dbgprintf("keep-alive set to %d\n", iVal); + pThis->bUseKeepAlive = iVal; + RETiRet; +} + +static rsRetVal +SetOnCharRcvd(strmsrv_t *pThis, rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar)) +{ + DEFiRet; + assert(OnCharRcvd != NULL); + pThis->OnCharRcvd = OnCharRcvd; + RETiRet; +} + +/* Set the input name to use -- rgerhards, 2008-12-10 */ +static rsRetVal +SetInputName(strmsrv_t *pThis, uchar *name) +{ + uchar *pszName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + if(name == NULL) + pszName = NULL; + else + CHKmalloc(pszName = ustrdup(name)); + free(pThis->pszInputName); + pThis->pszInputName = pszName; +finalize_it: + RETiRet; +} + + +/* here follows a number of methods that shuffle authentication settings down + * to the drivers. Drivers not supporting these settings may return an error + * state. + * -------------------------------------------------------------------------- */ + +/* set the driver mode -- rgerhards, 2008-04-30 */ +static rsRetVal +SetDrvrMode(strmsrv_t *pThis, int iMode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iDrvrMode = iMode; + RETiRet; +} + + +/* set the driver authentication mode -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrAuthMode(strmsrv_t *pThis, uchar *mode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + CHKmalloc(pThis->pszDrvrAuthMode = ustrdup(mode)); +finalize_it: + RETiRet; +} + + +/* set the driver's permitted peers -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrPermPeers(strmsrv_t *pThis, permittedPeers_t *pPermPeers) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->pPermPeers = pPermPeers; + RETiRet; +} + + +/* End of methods to shuffle autentication settings to the driver.; + + * -------------------------------------------------------------------------- */ + + +/* set max number of sessions + * this must be called before ConstructFinalize, or it will have no effect! + * rgerhards, 2009-04-09 + */ +static rsRetVal +SetSessMax(strmsrv_t *pThis, int iMax) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iSessMax = iMax; + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strmsrv) +CODESTARTobjQueryInterface(strmsrv) + if(pIf->ifVersion != strmsrvCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strmsrvDebugPrint; + pIf->Construct = strmsrvConstruct; + pIf->ConstructFinalize = strmsrvConstructFinalize; + pIf->Destruct = strmsrvDestruct; + + pIf->configureSTRMListen = configureSTRMListen; + pIf->create_strm_socket = create_strm_socket; + pIf->Run = Run; + + pIf->SetKeepAlive = SetKeepAlive; + pIf->SetUsrP = SetUsrP; + pIf->SetInputName = SetInputName; + pIf->SetSessMax = SetSessMax; + pIf->SetDrvrMode = SetDrvrMode; + pIf->SetDrvrAuthMode = SetDrvrAuthMode; + pIf->SetDrvrPermPeers = SetDrvrPermPeers; + pIf->SetCBIsPermittedHost = SetCBIsPermittedHost; + pIf->SetCBOpenLstnSocks = SetCBOpenLstnSocks; + pIf->SetCBOnSessAccept = SetCBOnSessAccept; + pIf->SetCBOnSessConstructFinalize = SetCBOnSessConstructFinalize; + pIf->SetCBOnSessDestruct = SetCBOnSessDestruct; + pIf->SetCBOnDestruct = SetCBOnDestruct; + pIf->SetCBOnRegularClose = SetCBOnRegularClose; + pIf->SetCBOnErrClose = SetCBOnErrClose; + pIf->SetOnCharRcvd = SetOnCharRcvd; + +finalize_it: +ENDobjQueryInterface(strmsrv) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strmsrv, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strmsrv) + /* release objects we no longer need */ + objRelease(strms_sess, DONT_LOAD_LIB); + objRelease(conf, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrms, DONT_LOAD_LIB); + objRelease(nssel, DONT_LOAD_LIB); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(net, LM_NET_FILENAME); +ENDObjClassExit(strmsrv) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strmsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(net, LM_NET_FILENAME)); + CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); + CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(strms_sess, DONT_LOAD_LIB)); + CHKiRet(objUse(conf, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strmsrvDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmsrvConstructFinalize); +ENDObjClassInit(strmsrv) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit + /* de-init in reverse order! */ + strmsrvClassExit(); + strms_sessClassExit(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + /* Initialize all classes that are in our module - this includes ourselfs */ + CHKiRet(strms_sessClassInit(pModInfo)); + CHKiRet(strmsrvClassInit(pModInfo)); /* must be done after strms_sess, as we use it */ +ENDmodInit + +/* vim:set ai: + */ diff --git a/runtime/strmsrv.h b/runtime/strmsrv.h new file mode 100644 index 00000000..86e529c3 --- /dev/null +++ b/runtime/strmsrv.h @@ -0,0 +1,112 @@ +/* Definitions for strmsrv class. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMSRV_H +#define INCLUDED_STRMSRV_H + +#include "obj.h" +#include "strms_sess.h" + +/* list of strm listen ports */ +struct strmLstnPortList_s { + uchar *pszPort; /**< the ports the listener shall listen on */ + uchar *pszInputName; /**< value to be used as input name */ + strmsrv_t *pSrv; /**< pointer to higher-level server instance */ + strmLstnPortList_t *pNext; /**< next port or NULL */ +}; + + +/* the strmsrv object */ +struct strmsrv_s { + BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ + int bUseKeepAlive; /**< use socket layer KEEPALIVE handling? */ + netstrms_t *pNS; /**< pointer to network stream subsystem */ + int iDrvrMode; /**< mode of the stream driver to use */ + uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ + uchar *pszInputName; /**< value to be used as input name */ + permittedPeers_t *pPermPeers;/**< driver's permitted peers */ + int iLstnMax; /**< max nbr of listeners currently supported */ + netstrm_t **ppLstn; /**< our netstream listners */ + strmLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ + int iSessMax; /**< max number of sessions supported */ + strmLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ + int addtlFrameDelim; /**< additional frame delimiter for plain STRM syslog framing (e.g. to handle NetScreen) */ + strms_sess_t **pSessions;/**< array of all of our sessions */ + void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ + /* callbacks */ + int (*pIsPermittedHost)(struct sockaddr *addr, char *fromHostFQDN, void*pUsrSrv, void*pUsrSess); + rsRetVal (*pRcvData)(strms_sess_t*, char*, size_t, ssize_t *); + rsRetVal (*OpenLstnSocks)(struct strmsrv_s*); + rsRetVal (*pOnListenDeinit)(void*); + rsRetVal (*OnDestruct)(void*); + rsRetVal (*pOnRegularClose)(strms_sess_t *pSess); + rsRetVal (*pOnErrClose)(strms_sess_t *pSess); + /* session specific callbacks */ + rsRetVal (*pOnSessAccept)(strmsrv_t *, strms_sess_t*); + rsRetVal (*OnSessConstructFinalize)(void*); + rsRetVal (*pOnSessDestruct)(void*); + rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar); +}; + + +/* interfaces */ +BEGINinterface(strmsrv) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strmsrv); + rsRetVal (*Construct)(strmsrv_t **ppThis); + rsRetVal (*ConstructFinalize)(strmsrv_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strmsrv_t **ppThis); + rsRetVal (*configureSTRMListen)(strmsrv_t*, uchar *pszPort); + //rsRetVal (*SessAccept)(strmsrv_t *pThis, strmLstnPortList_t*, strms_sess_t **ppSess, netstrm_t *pStrm); + rsRetVal (*create_strm_socket)(strmsrv_t *pThis); + rsRetVal (*Run)(strmsrv_t *pThis); + /* set methods */ + rsRetVal (*SetAddtlFrameDelim)(strmsrv_t*, int); + rsRetVal (*SetInputName)(strmsrv_t*, uchar*); + rsRetVal (*SetKeepAlive)(strmsrv_t*, int); + rsRetVal (*SetUsrP)(strmsrv_t*, void*); + rsRetVal (*SetCBIsPermittedHost)(strmsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); + rsRetVal (*SetCBOpenLstnSocks)(strmsrv_t *, rsRetVal (*)(strmsrv_t*)); + rsRetVal (*SetCBOnDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnRegularClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetCBOnErrClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetDrvrMode)(strmsrv_t *pThis, int iMode); + rsRetVal (*SetDrvrAuthMode)(strmsrv_t *pThis, uchar *pszMode); + rsRetVal (*SetDrvrPermPeers)(strmsrv_t *pThis, permittedPeers_t*); + /* session specifics */ + rsRetVal (*SetCBOnSessAccept)(strmsrv_t*, rsRetVal (*) (strmsrv_t*, strms_sess_t*)); + rsRetVal (*SetCBOnSessDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnSessConstructFinalize)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetSessMax)(strmsrv_t *pThis, int iMaxSess); + rsRetVal (*SetOnCharRcvd)(strmsrv_t *pThis, rsRetVal (*OnMsgCharRcvd)(strms_sess_t*, uchar)); +ENDinterface(strmsrv) +#define strmsrvCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* change for v?: + */ + + +/* prototypes */ +PROTOTYPEObj(strmsrv); + +/* the name of our library binary */ +#define LM_STRMSRV_FILENAME "lmstrmsrv" + +#endif /* #ifndef INCLUDED_STRMSRV_H */ diff --git a/runtime/sync.c b/runtime/sync.c index a3053e28..15d5c80f 100644 --- a/runtime/sync.c +++ b/runtime/sync.c @@ -28,12 +28,13 @@ #include "rsyslog.h" #include "sync.h" +#include "debug.h" void SyncObjInit(pthread_mutex_t **mut) { - *mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + *mut = (pthread_mutex_t *) MALLOC(sizeof (pthread_mutex_t)); pthread_mutex_init(*mut, NULL); } diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h index be0dfdd8..fcebd985 100644 --- a/runtime/syslogd-types.h +++ b/runtime/syslogd-types.h @@ -56,7 +56,10 @@ * applications I do not yet envision. -- rgerhards, 2007-07-24 */ typedef enum _syslogFeature { - sFEATURERepeatedMsgReduction = 1 + sFEATURERepeatedMsgReduction = 1, /* for output modules */ + sFEATURENonCancelInputTermination = 2, /* for input modules */ + sFEATUREAutomaticSanitazion = 3, /* for parser modules */ + sFEATUREAutomaticPRIParsing = 4 /* for parser modules */ } syslogFeature; /* we define our own facility and severities */ @@ -76,25 +79,31 @@ enum _EHostnameCmpMode { }; typedef enum _EHostnameCmpMode EHostnameCmpMode; +/* time type numerical values for structure below */ +#define TIME_TYPE_UNINIT 0 +#define TIME_TYPE_RFC3164 1 +#define TIME_TYPE_RFC5424 2 /* rgerhards 2004-11-11: the following structure represents * a time as it is used in syslog. + * rgerhards, 2009-06-23: packed structure for better cache performance + * (but left ultimate decision about packing to compiler) */ struct syslogTime { - int timeType; /* 0 - unitinialized , 1 - RFC 3164, 2 - syslog-protocol */ - int year; - int month; - int day; - int hour; /* 24 hour clock */ - int minute; - int second; - int secfrac; /* fractional seconds (must be 32 bit!) */ - int secfracPrecision; + intTiny timeType; /* 0 - unitinialized , 1 - RFC 3164, 2 - syslog-protocol */ + intTiny month; + intTiny day; + intTiny hour; /* 24 hour clock */ + intTiny minute; + intTiny second; + intTiny secfracPrecision; + intTiny OffsetMinute; /* UTC offset in minutes */ + intTiny OffsetHour; /* UTC offset in hours + * full UTC offset minutes = OffsetHours*60 + OffsetMinute. Then use + * OffsetMode to know the direction. + */ char OffsetMode; /* UTC offset + or - */ - char OffsetHour; /* UTC offset in hours */ - int OffsetMinute; /* UTC offset in minutes */ - /* full UTC offset minutes = OffsetHours*60 + OffsetMinute. Then use - * OffsetMode to know the direction. - */ + short year; + int secfrac; /* fractional seconds (must be 32 bit!) */ }; typedef struct syslogTime syslogTime_t; diff --git a/runtime/sysvar.c b/runtime/sysvar.c index c102d1f5..4a6ace19 100644 --- a/runtime/sysvar.c +++ b/runtime/sysvar.c @@ -175,8 +175,6 @@ CODESTARTobjQueryInterface(sysvar) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - //xxxpIf->oID = "sysvar";//OBJsysvar; - pIf->Construct = sysvarConstruct; pIf->ConstructFinalize = sysvarConstructFinalize; pIf->Destruct = sysvarDestruct; diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 36d76a78..7a776f68 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -4,6 +4,9 @@ * The following functions are wrappers which hopefully enable us to move * from 8-bit chars to unicode with relative ease when we finally attack this * + * Note: while we prefer inline functions, this leads to invalid references in + * core dumps. So in a debug build, we use macros where appropriate... + * * Begun 2009-05-21 RGerhards * * Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH @@ -31,6 +34,22 @@ #include <string.h> +#ifdef DEBUG +# define ustrncpy(psz1, psz2, len) strncpy((char*)(psz1), (char*)(psz2), (len)) +# define ustrdup(psz) (uchar*)strdup((char*)(psz)) +#else + static inline uchar* ustrncpy(uchar *psz1, uchar *psz2, size_t len) + { + return (uchar*) strncpy((char*) psz1, (char*) psz2, len); + } + + static inline uchar* ustrdup(uchar *psz) + { + return (uchar*) strdup((char*)psz); + } + +#endif /* #ifdef DEBUG */ + static inline int ustrcmp(uchar *psz1, uchar *psz2) { return strcmp((char*) psz1, (char*) psz2); @@ -41,13 +60,9 @@ static inline int ustrlen(uchar *psz) return strlen((char*) psz); } -static inline uchar* ustrdup(uchar *psz) -{ - return (uchar*) strdup((char*)psz); -} - #define UCHAR_CONSTANT(x) ((uchar*) (x)) +#define CHAR_CONVERT(x) ((char*) (x)) #endif /* multi-include protection */ /* vim:set ai: diff --git a/runtime/unlimited_select.h b/runtime/unlimited_select.h new file mode 100644 index 00000000..32dadc03 --- /dev/null +++ b/runtime/unlimited_select.h @@ -0,0 +1,45 @@ +/* unlimited_select.h + * Tweak the macros for accessing fd_set so that the select() syscall + * won't be limited to a particular number of file descriptors. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +#ifndef UNLIMITED_SELECT_H_INCLUDED + +#include <string.h> +#include <stdlib.h> +#include <sys/select.h> +#include "glbl.h" + +#ifdef USE_UNLIMITED_SELECT +# undef FD_ZERO +# define FD_ZERO(set) memset((set), 0, glbl.GetFdSetSize()); +#endif + +#ifdef USE_UNLIMITED_SELECT +void freeFdSet(fd_set *p) { + free(p); +} +#else +# define freeFdSet(x) +#endif + +#endif /* #ifndef UNLIMITED_SELECT_H_INCLUDED */ diff --git a/runtime/vm.c b/runtime/vm.c index 125b0d21..0ed174d1 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -34,6 +34,7 @@ #include "vm.h" #include "sysvar.h" #include "stringbuf.h" +#include "unicode-helper.h" /* static data */ DEFobjStaticHelpers @@ -41,6 +42,8 @@ DEFobjCurrIf(vmstk) DEFobjCurrIf(var) DEFobjCurrIf(sysvar) +static pthread_mutex_t mutGetenv; /* we need to make this global because otherwise we can not guarantee proper init! */ + /* ------------------------------ function registry code and structures ------------------------------ */ /* we maintain a registry of known functions */ @@ -82,11 +85,15 @@ rsfrAddFunction(uchar *szName, prsf_t rsf) /* unique name, so add to head of list */ CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + CHKiRet(cstrFinalize(pEntry->pName)); pEntry->rsf = rsf; pEntry->pNext = funcRegRoot; funcRegRoot = pEntry; finalize_it: + if(iRet != RS_RET_OK && iRet != RS_RET_DUP_FUNC_NAME) + free(pEntry); + RETiRet; } @@ -167,7 +174,7 @@ rsfrRemoveAll(void) while(pEntry != NULL) { pEntryDel = pEntry; pEntry = pEntry->pNext; - rsCStrDestruct(&pEntryDel->pName); + cstrDestruct(&pEntryDel->pName); free(pEntryDel); } funcRegRoot = NULL; @@ -405,6 +412,7 @@ CODESTARTop(STRADD) vmstk.PopString(pThis->pStk, &operand1); CHKiRet(rsCStrAppendCStr(operand1->val.pStr, operand2->val.pStr)); + CHKiRet(cstrFinalize(operand1->val.pStr)); /* we have a result, so let's push it */ vmstk.Push(pThis->pStk, operand1); @@ -537,6 +545,42 @@ finalize_it: } +/* The getenv function. Note that we guard the OS call by a mutex, as that + * function is not guaranteed to be thread-safe. This implementation here is far from + * being optimal, at least we should cache the result. This is left TODO for + * a later revision. + * rgerhards, 2009-11-03 + */ +static rsRetVal +rsf_getenv(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + char *envResult; + cstr_t *pCstr; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton (trivial case here...) */ + vmstk.PopString(pStk, &operand1); + d_pthread_mutex_lock(&mutGetenv); + envResult = getenv((char*) rsCStrGetSzStr(operand1->val.pStr)); + DBGPRINTF("rsf_getenv(): envvar '%s', return '%s'\n", rsCStrGetSzStr(operand1->val.pStr), + envResult == NULL ? "(NULL)" : envResult); + iRet = rsCStrConstructFromszStr(&pCstr, (envResult == NULL) ? UCHAR_CONSTANT("") : (uchar*)envResult); + d_pthread_mutex_unlock(&mutGetenv); + if(iRet != RS_RET_OK) + FINALIZE; /* need to do this after mutex is unlocked! */ + + /* Store result and cleanup */ + var.SetString(operand1, pCstr); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + /* The "tolower" function, which converts its sole argument to lower case. * Quite honestly, currently this is primarily a test driver for me... * rgerhards, 2009-04-06 @@ -554,16 +598,16 @@ rsf_tolower(vmstk_t *pStk, int numOperands) ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); /* pop args and do operaton */ - CHKiRet(rsCStrConstruct(&pcstr)); + CHKiRet(cstrConstruct(&pcstr)); vmstk.PopString(pStk, &operand1); - pSrc = rsCStrGetSzStr(operand1->val.pStr); - iStrlen = strlen((char*)pSrc); + pSrc = cstrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); // TODO: use count from string! while(iStrlen--) { - CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + CHKiRet(cstrAppendChar(pcstr, tolower(*pSrc++))); } /* Store result and cleanup */ - CHKiRet(rsCStrFinish(pcstr)); + CHKiRet(cstrFinalize(pcstr)); var.SetString(operand1, pcstr); vmstk.Push(pStk, operand1); finalize_it: @@ -754,6 +798,8 @@ BEGINObjClassExit(vm, OBJ_IS_CORE_MODULE) /* class, version */ objRelease(sysvar, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(vmstk, CORE_COMPONENT); + + pthread_mutex_destroy(&mutGetenv); ENDObjClassExit(vm) @@ -774,6 +820,9 @@ BEGINObjClassInit(vm, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* register built-in functions // TODO: move to its own module */ CHKiRet(rsfrAddFunction((uchar*)"strlen", rsf_strlen)); CHKiRet(rsfrAddFunction((uchar*)"tolower", rsf_tolower)); + CHKiRet(rsfrAddFunction((uchar*)"getenv", rsf_getenv)); + + pthread_mutex_init(&mutGetenv, NULL); ENDObjClassInit(vm) diff --git a/runtime/vmop.c b/runtime/vmop.c index 3e001d27..ea627220 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -85,7 +85,7 @@ CODESTARTobjDebugPrint(vmop) CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); } } - CHKiRet(rsCStrFinish(&pStrVar)); + CHKiRet(cstrFinalize(pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); if(pThis->opcode != opcode_FUNC_CALL) rsCStrDestruct(&pStrVar); @@ -125,7 +125,7 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) if(pThis->operand.pVar != NULL) CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); } - CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); + CHKiRet(cstrAppendChar(pstrPrg, '\n')); finalize_it: RETiRet; diff --git a/runtime/wti.c b/runtime/wti.c index 544bffa7..14964fb0 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -39,10 +39,10 @@ #include <pthread.h> #include <errno.h> -#ifdef OS_SOLARIS -# include <sched.h> -# define pthread_yield() sched_yield() -#endif +/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 +//#ifdef OS_SOLARIS +//# include <sched.h> +//#endif #include "rsyslog.h" #include "stringbuf.h" @@ -51,6 +51,7 @@ #include "wti.h" #include "obj.h" #include "glbl.h" +#include "atomic.h" /* static data */ DEFobjStaticHelpers @@ -75,85 +76,50 @@ wtiGetDbgHdr(wti_t *pThis) } -/* get the current worker state. For simplicity and speed, we have - * NOT used our regular calling interface this time. I hope that won't - * bite in the long term... -- rgerhards, 2008-01-17 - * TODO: may be performance optimized by atomic operations +/* return the current worker processing state. For the sake of + * simplicity, we do not use the iRet interface. -- rgerhards, 2009-07-17 */ -qWrkCmd_t -wtiGetState(wti_t *pThis, int bLockMutex) +sbool +wtiGetState(wti_t *pThis) { - DEFVARS_mutexProtection; - qWrkCmd_t tCmd; - - BEGINfunc - ISOBJ_TYPE_assert(pThis, wti); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - tCmd = pThis->tCurrCmd; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - ENDfunc - return tCmd; + return ATOMIC_FETCH_32BIT(pThis->bIsRunning); } -/* send a command to a specific thread - * bActiveOnly specifies if the command should be sent only when the worker is - * in an active state. -- rgerhards, 2008-01-20 +/* Set this thread to "always running" state (can not be unset) + * rgerhards, 2009-07-20 */ rsRetVal -wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) +wtiSetAlwaysRunning(wti_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; - ISOBJ_TYPE_assert(pThis, wti); - assert(tCmd <= eWRKTHRD_SHUTDOWN_IMMEDIATE); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - - /* all worker states must be followed sequentially, only termination can be set in any state */ - if( (bActiveOnly && (pThis->tCurrCmd < eWRKTHRD_RUN_CREATED)) - || (pThis->tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) { - dbgprintf("%s: command %d can not be accepted in current %d processing state - ignored\n", - wtiGetDbgHdr(pThis), tCmd, pThis->tCurrCmd); - } else { - dbgprintf("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd); - /* we could replace this with a simple if, but we leave the switch in in case we need - * to add something at a later stage. -- rgerhards, 2008-09-30 - */ - switch(tCmd) { - case eWRKTHRD_TERMINATING: - /* TODO: re-enable meaningful debug msg! (via function callback?) - dbgprintf("%s: thread terminating with %d entries left in queue, %d workers running.\n", - wtiGetDbgHdr(pThis->pQueue), pThis->pQueue->iQueueSize, - pThis->pQueue->iCurNumWrkThrd); - */ - pthread_cond_signal(&pThis->condExitDone); - dbgprintf("%s: worker terminating\n", wtiGetDbgHdr(pThis)); - break; - /* these cases just to satisfy the compiler, we do (yet) not act an them: */ - case eWRKTHRD_RUNNING: - case eWRKTHRD_STOPPED: - case eWRKTHRD_RUN_CREATED: - case eWRKTHRD_RUN_INIT: - case eWRKTHRD_SHUTDOWN: - case eWRKTHRD_SHUTDOWN_IMMEDIATE: - /* DO NOTHING */ - break; - } - pThis->tCurrCmd = tCmd; /* apply the new state */ - } + pThis->bAlwaysRunning = TRUE; + return RS_RET_OK; +} - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - RETiRet; +/* Set status (thread is running or not), actually an property of + * use for wtp, but we need to have it per thread instance (thus it + * is inside wti). -- rgerhards, 2009-07-17 + */ +rsRetVal +wtiSetState(wti_t *pThis, sbool bNewVal) +{ + ISOBJ_TYPE_assert(pThis, wti); + if(bNewVal) + ATOMIC_STORE_1_TO_INT(pThis->bIsRunning); + else + ATOMIC_STORE_0_TO_INT(pThis->bIsRunning); + return RS_RET_OK; } -/* Cancel the thread. If the thread is already cancelled or termination, - * we do not again cancel it. But it is save and legal to call wtiCancelThrd() in - * such situations. +/* Cancel the thread. If the thread is not running. But it is save and legal to + * call wtiCancelThrd() in such situations. This function only returns when the + * thread has terminated. Else we may get race conditions all over the code... + * Note that when waiting for the thread to terminate, we do a busy wait, checking + * progress every 10ms. It is very unlikely that we will ever cancel a thread + * and, if so, it will only happen at the end of the rsyslog run. So doing this + * kind of non-optimal wait is considered preferable over using condition variables. * rgerhards, 2008-02-26 */ rsRetVal @@ -163,17 +129,15 @@ wtiCancelThrd(wti_t *pThis) ISOBJ_TYPE_assert(pThis, wti); - d_pthread_mutex_lock(&pThis->mut); - - if(pThis->tCurrCmd >= eWRKTHRD_TERMINATING) { + if(wtiGetState(pThis)) { dbgoprint((obj_t*) pThis, "canceling worker thread\n"); pthread_cancel(pThis->thrdID); - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pThis->pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + /* now wait until the thread terminates... */ + while(wtiGetState(pThis)) { + srSleep(0, 10000); + } } - d_pthread_mutex_unlock(&pThis->mut); - RETiRet; } @@ -181,37 +145,15 @@ wtiCancelThrd(wti_t *pThis) /* Destructor */ BEGINobjDestruct(wti) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(wti) - /* if we reach this point, we must make sure the associated worker has terminated. It is - * the callers duty to make sure the worker already knows it shall terminate. - * TODO: is it *really* the caller's duty? ...mmmhhhh.... smells bad... rgerhards, 2008-01-25 - */ - wtiProcessThrdChanges(pThis, LOCK_MUTEX); /* process state change one last time */ - - d_pthread_mutex_lock(&pThis->mut); - if(wtiGetState(pThis, MUTEX_ALREADY_LOCKED) != eWRKTHRD_STOPPED) { - dbgprintf("%s: WARNING: worker %p shall be destructed but is still running (might be OK) - joining it\n", - wtiGetDbgHdr(pThis), pThis); - /* let's hope the caller actually instructed it to shutdown... */ - pthread_cond_wait(&pThis->condExitDone, &pThis->mut); - wtiJoinThrd(pThis); - } - d_pthread_mutex_unlock(&pThis->mut); - /* actual destruction */ - pthread_cond_destroy(&pThis->condExitDone); - pthread_mutex_destroy(&pThis->mut); - - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->batch.pElem); + free(pThis->pszDbgHdr); ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); - pthread_cond_init(&pThis->condExitDone, NULL); - pthread_mutex_init(&pThis->mut, NULL); ENDobjConstruct(wti) @@ -222,81 +164,28 @@ rsRetVal wtiConstructFinalize(wti_t *pThis) { DEFiRet; + int iDeqBatchSize; ISOBJ_TYPE_assert(pThis, wti); dbgprintf("%s: finalizing construction of worker instance data\n", wtiGetDbgHdr(pThis)); - /* initialize our thread instance descriptor */ - pThis->pUsrp = NULL; - pThis->tCurrCmd = eWRKTHRD_STOPPED; - - RETiRet; -} - - -/* join a specific worker thread - * we do not lock the mutex, because join will sync anyways... - */ -rsRetVal -wtiJoinThrd(wti_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, wti); - dbgprintf("waiting for worker %s termination, current state %d\n", wtiGetDbgHdr(pThis), pThis->tCurrCmd); - if (pThis->thrdID == 0) { - dbgprintf("worker %s was already stopped\n", wtiGetDbgHdr(pThis)); - } else { - pthread_join(pThis->thrdID, NULL); - wtiSetState(pThis, eWRKTHRD_STOPPED, 0, MUTEX_ALREADY_LOCKED); /* back to virgin... */ - pThis->thrdID = 0; /* invalidate the thread ID so that we do not accidently find reused ones */ - dbgprintf("worker %s has stopped\n", wtiGetDbgHdr(pThis)); - } - - RETiRet; -} - -/* check if we had a worker thread changes and, if so, act - * on it. At a minimum, terminated threads are harvested (joined). - */ -rsRetVal -wtiProcessThrdChanges(wti_t *pThis, int bLockMutex) -{ - DEFiRet; - DEFVARS_mutexProtection; + /* initialize our thread instance descriptor (no concurrency here) */ + pThis->bIsRunning = FALSE; - ISOBJ_TYPE_assert(pThis, wti); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - switch(pThis->tCurrCmd) { - case eWRKTHRD_TERMINATING: - /* we need to at least temporarily release the mutex, because otherwise - * we may deadlock with the thread we intend to join (it aquires the mutex - * during termination processing). -- rgerhards, 2008-02-26 - */ - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - iRet = wtiJoinThrd(pThis); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - break; - /* these cases just to satisfy the compiler, we do not act an them: */ - case eWRKTHRD_STOPPED: - case eWRKTHRD_RUN_CREATED: - case eWRKTHRD_RUN_INIT: - case eWRKTHRD_RUNNING: - case eWRKTHRD_SHUTDOWN: - case eWRKTHRD_SHUTDOWN_IMMEDIATE: - /* DO NOTHING */ - break; - } - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ + CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize)); + CHKmalloc(pThis->batch.pElem = calloc((size_t)iDeqBatchSize, sizeof(batch_obj_t))); +finalize_it: RETiRet; } /* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. + * Most importantly, it must bring back the batch into a consistent state. + * Keep in mind that cancellation is disabled if we run into + * the cancel cleanup handler (and have been cancelled). * rgerhards, 2008-01-16 */ static void @@ -304,7 +193,6 @@ wtiWorkerCancelCleanup(void *arg) { wti_t *pThis = (wti_t*) arg; wtp_t *pWtp; - int iCancelStateSave; BEGINfunc ISOBJ_TYPE_assert(pThis, wti); @@ -312,129 +200,108 @@ wtiWorkerCancelCleanup(void *arg) ISOBJ_TYPE_assert(pWtp, wtp); DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); + pWtp->pfObjProcessed(pWtp->pUsr, pThis); + DBGPRINTF("%s: done cancelation cleanup handler.\n", wtiGetDbgHdr(pThis)); - /* call user supplied handler (that one e.g. requeues the element) */ - pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp); + ENDfunc +} - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pWtp->mut); - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - /* TODO: sync access? I currently think it is NOT needed -- rgerhards, 2008-01-28 */ - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ - d_pthread_mutex_unlock(&pWtp->mut); - pthread_setcancelstate(iCancelStateSave, NULL); +/* wait for queue to become non-empty or timeout + * helper to wtiWorker. Note the the predicate is + * re-tested by the caller, so it is OK to NOT do it here. + * rgerhards, 2009-05-20 + */ +static inline void +doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) +{ + struct timespec t; + + BEGINfunc + DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); + + if(pThis->bAlwaysRunning) { + /* never shut down any started worker */ + d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); + } else { + timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ + if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { + DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); + *pbInactivityTOOccured = 1; /* indicate we had a timeout */ + } + } + dbgoprint((obj_t*) pThis, "worker awoke from idle processing\n"); ENDfunc } -/* generic worker thread framework - * - * Some special comments below, so that they do not clutter the main function code: - * - * On the use of pthread_testcancel(): - * Now make sure we can get canceled - it is not specified if pthread_setcancelstate() is - * a cancellation point in itself. As we run most of the time without cancel enabled, I fear - * we may never get cancelled if we do not create a cancellation point ourselfs. - * - * On the use of pthread_yield(): - * We yield to give the other threads a chance to obtain the mutex. If we do not - * do that, this thread may very well aquire the mutex again before another thread - * has even a chance to run. The reason is that mutex operations are free to be - * implemented in the quickest possible way (and they typically are!). That is, the - * mutex lock/unlock most probably just does an atomic memory swap and does not necessarily - * schedule other threads waiting on the same mutex. That can lead to the same thread - * aquiring the mutex ever and ever again while all others are starving for it. We - * have exactly seen this behaviour when we deliberately introduced a long-running - * test action which basically did a sleep. I understand that with real actions the - * likelihood of this starvation condition is very low - but it could still happen - * and would be very hard to debug. The yield() is a sure fix, its performance overhead - * should be well accepted given the above facts. -- rgerhards, 2008-01-10 +/* generic worker thread framework. Note that we prohibit cancellation + * during almost all times, because it can have very undesired side effects. + * However, we may need to cancel a thread if the consumer blocks for too + * long (during shutdown). So what we do is block cancellation, and every + * consumer must enable it during the periods where it is safe. */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal wtiWorker(wti_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; - struct timespec t; wtp_t *pWtp; /* our worker thread pool */ int bInactivityTOOccured = 0; + rsRetVal localRet; + rsRetVal terminateRet; + int iCancelStateSave; + DEFiRet; ISOBJ_TYPE_assert(pThis, wti); pWtp = pThis->pWtp; /* shortcut */ ISOBJ_TYPE_assert(pWtp, wtp); dbgSetThrdName(pThis->pszDbgHdr); - pThis->pUsrp = NULL; pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - pWtp->pfOnWorkerStartup(pWtp->pUsr); - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); /* now we have our identity, on to real processing */ while(1) { /* loop will be broken below - need to do mutex locks */ - /* process any pending thread requests */ - wtpProcessThrdChanges(pWtp); - pthread_testcancel(); /* see big comment in function header */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); /* see big comment in function header */ -# endif - - /* if we have a rate-limiter set for this worker pool, let's call it. Please - * keep in mind that the rate-limiter may hold us for an extended period - * of time. -- rgerhards, 2008-04-02 - */ - if(pWtp->pfRateLimiter != NULL) { + if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */ pWtp->pfRateLimiter(pWtp->pUsr); } - wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - - if( (bInactivityTOOccured && pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) - || wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED)) { - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); - break; /* end worker thread run */ + d_pthread_mutex_lock(pWtp->pmutUsr); + + /* first check if we are in shutdown process (but evaluate a bit later) */ + terminateRet = wtpChkStopWrkr(pWtp, MUTEX_ALREADY_LOCKED); + if(terminateRet == RS_RET_TERMINATE_NOW) { + /* we now need to free the old batch */ + localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis); + dbgoprint((obj_t*) pThis, "terminating worker because of TERMINATE_NOW mode, del iRet %d\n", + localRet); + d_pthread_mutex_unlock(pWtp->pmutUsr); + break; } - bInactivityTOOccured = 0; /* reset for next run */ - /* if we reach this point, we are still protected by the mutex */ - - if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) { - DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); - pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); - - if(pWtp->toWrkShutdown == -1) { - /* never shut down any started worker */ - d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); - } else { - timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ - if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { - DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); - bInactivityTOOccured = 1; /* indicate we had a timeout */ - } + /* try to execute and process whatever we have */ + /* Note that this function releases and re-aquires the mutex. The returned + * information on idle state must be processed before releasing the mutex again. + */ + localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); + + if(localRet == RS_RET_IDLE) { + if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { + d_pthread_mutex_unlock(pWtp->pmutUsr); + break; /* end of loop */ } - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); + d_pthread_mutex_unlock(pWtp->pmutUsr); continue; /* request next iteration */ } - /* if we reach this point, we have a non-empty queue (and are still protected by mutex) */ - pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); + d_pthread_mutex_unlock(pWtp->pmutUsr); + + bInactivityTOOccured = 0; /* reset for next run */ } /* indicate termination */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pThis->mut); pthread_cleanup_pop(0); /* remove cleanup handler */ - - pWtp->pfOnWorkerShutdown(pWtp->pUsr); - - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ - d_pthread_mutex_unlock(&pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); RETiRet; @@ -463,10 +330,9 @@ wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg) if(pThis->pszDbgHdr != NULL) { free(pThis->pszDbgHdr); - pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ @@ -497,6 +363,5 @@ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wti) -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/wti.h b/runtime/wti.h index 6b60b833..e587c69e 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -1,6 +1,6 @@ /* Definition of the worker thread instance (wti) class. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 by Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -27,22 +27,19 @@ #include <pthread.h> #include "wtp.h" #include "obj.h" +#include "batch.h" + /* the worker thread instance class */ -typedef struct wti_s { +struct wti_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ - pthread_t thrdID; /* thread ID */ - qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ + pthread_t thrdID; /* thread ID */ + int bIsRunning; /* is this thread currently running? (must be int for atomic op!) */ + sbool bAlwaysRunning; /* should this thread always run? */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ - pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */ - pthread_mutex_t mut; - int bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ + batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ uchar *pszDbgHdr; /* header string for debug messages */ -} wti_t; - -/* some symbolic constants for easier reference */ +}; /* prototypes */ @@ -50,12 +47,11 @@ rsRetVal wtiConstruct(wti_t **ppThis); rsRetVal wtiConstructFinalize(wti_t *pThis); rsRetVal wtiDestruct(wti_t **ppThis); rsRetVal wtiWorker(wti_t *pThis); -rsRetVal wtiProcessThrdChanges(wti_t *pThis, int bLockMutex); rsRetVal wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg); -rsRetVal wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex); -rsRetVal wtiJoinThrd(wti_t *pThis); rsRetVal wtiCancelThrd(wti_t *pThis); -qWrkCmd_t wtiGetState(wti_t *pThis, int bLockMutex); +rsRetVal wtiSetAlwaysRunning(wti_t *pThis); +rsRetVal wtiSetState(wti_t *pThis, sbool bNew); +sbool wtiGetState(wti_t *pThis); PROTOTYPEObjClassInit(wti); PROTOTYPEpropSetMeth(wti, pszDbgHdr, uchar*); PROTOTYPEpropSetMeth(wti, pWtp, wtp_t*); diff --git a/runtime/wtp.c b/runtime/wtp.c index 04eb974f..ab7ca4bb 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -8,7 +8,7 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -39,18 +39,23 @@ #include <fcntl.h> #include <unistd.h> #include <errno.h> - -#ifdef OS_SOLARIS -# include <sched.h> -# define pthread_yield() sched_yield() +#include <atomic.h> +#if HAVE_SYS_PRCTL_H +# include <sys/prctl.h> #endif +/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 +//#ifdef OS_SOLARIS +//# include <sched.h> +//#endif + #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" #include "wtp.h" #include "wti.h" #include "obj.h" +#include "unicode-helper.h" #include "glbl.h" /* static data */ @@ -78,22 +83,19 @@ wtpGetDbgHdr(wtp_t *pThis) /* Not implemented dummy function for constructor */ -static rsRetVal NotImplementedDummy() { return RS_RET_OK; } +static rsRetVal NotImplementedDummy() { return RS_RET_NOT_IMPLEMENTED; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); - pthread_mutex_init(&pThis->mut, NULL); - pthread_mutex_init(&pThis->mutThrdShutdwn, NULL); + pthread_mutex_init(&pThis->mutWtp, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); + pthread_attr_init(&pThis->attrThrd); + pthread_attr_setdetachstate(&pThis->attrThrd, PTHREAD_CREATE_DETACHED); /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; - pThis->pfIsIdle = NotImplementedDummy; + pThis->pfGetDeqBatchSize = NotImplementedDummy; pThis->pfDoWork = NotImplementedDummy; - pThis->pfOnIdle = NotImplementedDummy; - pThis->pfOnWorkerCancel = NotImplementedDummy; - pThis->pfOnWorkerStartup = NotImplementedDummy; - pThis->pfOnWorkerShutdown = NotImplementedDummy; + pThis->pfObjProcessed = NotImplementedDummy; ENDobjConstruct(wtp) @@ -111,13 +113,12 @@ wtpConstructFinalize(wtp_t *pThis) ISOBJ_TYPE_assert(pThis, wtp); - dbgprintf("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis)); + DBGPRINTF("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis)); /* alloc and construct workers - this can only be done in finalizer as we previously do * not know the max number of workers */ - if((pThis->pWrkr = malloc(sizeof(wti_t*) * pThis->iNumWorkerThreads)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - + CHKmalloc(pThis->pWrkr = MALLOC(sizeof(wti_t*) * pThis->iNumWorkerThreads)); + for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { CHKiRet(wtiConstruct(&pThis->pWrkr[i])); pWti = pThis->pWrkr[i]; @@ -137,8 +138,6 @@ finalize_it: BEGINobjDestruct(wtp) /* be sure to specify the object type also in END and CODESTART macros! */ int i; CODESTARTobjDestruct(wtp) - wtpProcessThrdChanges(pThis); /* process thread changes one last time */ - /* destruct workers */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) wtiDestruct(&pThis->pWrkr[i]); @@ -148,137 +147,68 @@ CODESTARTobjDestruct(wtp) /* actual destruction */ pthread_cond_destroy(&pThis->condThrdTrm); - pthread_mutex_destroy(&pThis->mut); - pthread_mutex_destroy(&pThis->mutThrdShutdwn); + pthread_mutex_destroy(&pThis->mutWtp); + pthread_attr_destroy(&pThis->attrThrd); - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->pszDbgHdr); ENDobjDestruct(wtp) -/* wake up at least one worker thread. - * rgerhards, 2008-01-20 - */ -rsRetVal -wtpWakeupWrkr(wtp_t *pThis) -{ - DEFiRet; - - /* TODO; mutex? I think not needed, as we do not need predictable exec order -- rgerhards, 2008-01-28 */ - ISOBJ_TYPE_assert(pThis, wtp); - pthread_cond_signal(pThis->pcondBusy); - RETiRet; -} - -/* wake up all worker threads. - * rgerhards, 2008-01-16 - */ -rsRetVal -wtpWakeupAllWrkr(wtp_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, wtp); - d_pthread_mutex_lock(pThis->pmutUsr); - pthread_cond_broadcast(pThis->pcondBusy); - d_pthread_mutex_unlock(pThis->pmutUsr); - RETiRet; -} - - -/* check if we had any worker thread changes and, if so, act - * on them. At a minimum, terminated threads are harvested (joined). - * This function MUST NEVER block on the queue mutex! - */ -rsRetVal -wtpProcessThrdChanges(wtp_t *pThis) -{ - DEFiRet; - int i; - - ISOBJ_TYPE_assert(pThis, wtp); - - if(pThis->bThrdStateChanged == 0) - FINALIZE; - - if(d_pthread_mutex_trylock(&(pThis->mutThrdShutdwn)) != 0) { - /* another thread is already in the loop */ - FINALIZE; - } - - /* Note: there is a left-over potential race condition below: - * pThis->bThrdStateChanged may be re-set by another thread while - * we work on it and thus the loop may terminate too early. However, - * there are no really bad effects from that so I perfer - for this - * version - to live with the problem as is. Not a good idea to - * introduce that large change into the stable branch without very - * good reason. -- rgerhards, 2009-04-02 - */ - do { - /* reset the change marker */ - pThis->bThrdStateChanged = 0; - /* go through all threads */ - for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - wtiProcessThrdChanges(pThis->pWrkr[i], LOCK_MUTEX); - } - /* restart if another change occured while we were processing the changes */ - } while(pThis->bThrdStateChanged != 0); - - d_pthread_mutex_unlock(&(pThis->mutThrdShutdwn)); - -finalize_it: - RETiRet; -} - - -/* Sent a specific state for the worker thread pool. - * rgerhards, 2008-01-21 +/* Sent a specific state for the worker thread pool. -- rgerhards, 2008-01-21 + * We do not need to do atomic instructions as set operations are only + * called when terminating the pool, and then in strict sequence. So we + * can never overwrite each other. On the other hand, it also doesn't + * matter if the read operation obtains an older value, as we then simply + * do one more iteration, what is perfectly legal (during shutdown + * they are awoken in any case). -- rgerhards, 2009-07-20 */ rsRetVal wtpSetState(wtp_t *pThis, wtpState_t iNewState) { - DEFiRet; - ISOBJ_TYPE_assert(pThis, wtp); pThis->wtpState = iNewState; - /* TODO: must wakeup workers? seen to be not needed -- rgerhards, 2008-01-28 */ - - RETiRet; + return RS_RET_OK; } /* check if the worker shall shutdown (1 = yes, 0 = no) - * TODO: check if we can use atomic operations to enhance performance * Note: there may be two mutexes locked, the bLockUsrMutex is the one in our "user" * (e.g. the queue clas) * rgerhards, 2008-01-21 */ rsRetVal -wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex) +wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex) { DEFiRet; - DEFVARS_mutexProtection; + wtpState_t wtpState; ISOBJ_TYPE_assert(pThis, wtp); + /* we need a consistent value, but it doesn't really matter if it is changed + * right after the fetch - then we simply do one more iteration in the worker + */ + wtpState = ATOMIC_FETCH_32BIT(pThis->wtpState); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - if( (pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) - || ((pThis->wtpState == wtpState_SHUTDOWN) && pThis->pfIsIdle(pThis->pUsr, bLockUsrMutex))) - iRet = RS_RET_TERMINATE_NOW; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + if(wtpState == wtpState_SHUTDOWN_IMMEDIATE) { + ABORT_FINALIZE(RS_RET_TERMINATE_NOW); + } else if(wtpState == wtpState_SHUTDOWN) { + ABORT_FINALIZE(RS_RET_TERMINATE_WHEN_IDLE); + } /* try customer handler if one was set and we do not yet have a definite result */ - if(iRet == RS_RET_OK && pThis->pfChkStopWrkr != NULL) { + if(pThis->pfChkStopWrkr != NULL) { iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex); } +finalize_it: RETiRet; } #pragma GCC diagnostic ignored "-Wempty-body" /* Send a shutdown command to all workers and see if they terminate. - * A timeout may be specified. + * A timeout may be specified. This function may also be called with + * the current number of workers being 0, in which case it does not + * shut down any worker. * rgerhards, 2008-01-14 */ rsRetVal @@ -286,30 +216,25 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout { DEFiRet; int bTimedOut; - int iCancelStateSave; ISOBJ_TYPE_assert(pThis, wtp); + /* lock mutex to prevent races (may otherwise happen during idle processing and such...) */ + d_pthread_mutex_lock(pThis->pmutUsr); wtpSetState(pThis, tShutdownCmd); - wtpWakeupAllWrkr(pThis); + pthread_cond_broadcast(pThis->pcondBusy); /* wake up all workers */ + d_pthread_mutex_unlock(pThis->pmutUsr); - /* see if we need to harvest (join) any terminated threads (even in timeout case, - * some may have terminated... - */ - wtpProcessThrdChanges(pThis); - - /* and wait for their termination */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pThis->mut); - pthread_cleanup_push(mutexCancelCleanup, &pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); + /* wait for worker thread termination */ + d_pthread_mutex_lock(&pThis->mutWtp); + pthread_cleanup_push(mutexCancelCleanup, &pThis->mutWtp); bTimedOut = 0; while(pThis->iCurNumWrkThrd > 0 && !bTimedOut) { - dbgprintf("%s: waiting %ldms on worker thread termination, %d still running\n", - wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), pThis->iCurNumWrkThrd); + DBGPRINTF("%s: waiting %ldms on worker thread termination, %d still running\n", + wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd)); - if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mut, ptTimeout) != 0) { - dbgprintf("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis)); + if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mutWtp, ptTimeout) != 0) { + DBGPRINTF("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis)); bTimedOut = 1; /* we exit the loop on timeout */ } } @@ -318,40 +243,11 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout if(bTimedOut) iRet = RS_RET_TIMED_OUT; - /* see if we need to harvest (join) any terminated threads (even in timeout case, - * some may have terminated... - */ - wtpProcessThrdChanges(pThis); - RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" -/* indicate that a thread has terminated and awake anyone waiting on it - * rgerhards, 2008-01-23 - */ -rsRetVal wtpSignalWrkrTermination(wtp_t *pThis) -{ - DEFiRet; - /* I leave the mutex code here out as it gives us deadlocks. I think it is not really - * needed and we are on the safe side. I leave this comment in if practice proves us - * wrong. The whole thing should be removed after half a year or year if we see there - * actually is no issue (or revisit it from a theoretical POV). - * rgerhards, 2008-01-28 - * revisited 2008-09-30, still a bit unclear, leave in - */ - /*TODO: mutex or not mutex, that's the question ;)DEFVARS_mutexProtection;*/ - - ISOBJ_TYPE_assert(pThis, wtp); - - /*BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX);*/ - pthread_cond_signal(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ - /*END_MTX_PROTECTED_OPERATIONS(&pThis->mut);*/ - RETiRet; -} - - /* Unconditionally cancel all running worker threads. * rgerhards, 2008-01-14 */ @@ -363,12 +259,8 @@ wtpCancelAll(wtp_t *pThis) ISOBJ_TYPE_assert(pThis, wtp); - /* process any pending thread requests so that we know who actually is still running */ - wtpProcessThrdChanges(pThis); - /* go through all workers and cancel those that are active */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - dbgprintf("%s: try canceling worker thread %d\n", wtpGetDbgHdr(pThis), i); wtiCancelThrd(pThis->pWrkr[i]); } @@ -376,40 +268,56 @@ wtpCancelAll(wtp_t *pThis) } - -/* Set the Inactivity Guard - * rgerhards, 2008-01-21 +/* this function contains shared code for both regular worker shutdown as + * well as shutdown via cancellation. We can not simply use pthread_cleanup_pop(1) + * as this introduces a race in the debug system (RETiRet system). + * rgerhards, 2009-10-26 */ -rsRetVal -wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex) +static inline void +wtpWrkrExecCleanup(wti_t *pWti) { - DEFiRet; - DEFVARS_mutexProtection; + wtp_t *pThis; - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - pThis->bInactivityGuard = bNewState; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + BEGINfunc + ISOBJ_TYPE_assert(pWti, wti); + pThis = pWti->pWtp; + ISOBJ_TYPE_assert(pThis, wtp); - RETiRet; + /* the order of the next two statements is important! */ + wtiSetState(pWti, WRKTHRD_STOPPED); + ATOMIC_DEC(pThis->iCurNumWrkThrd); + + DBGPRINTF("%s: Worker thread %lx, terminated, num workers now %d\n", + wtpGetDbgHdr(pThis), (unsigned long) pWti, ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd)); + + ENDfunc } -/* cancellation cleanup handler for executing worker - * decrements the worker counter - * rgerhards, 2008-01-20 +/* cancellation cleanup handler for executing worker decrements the worker counter. + * rgerhards, 2009-07-20 */ -void +static void wtpWrkrExecCancelCleanup(void *arg) { - wtp_t *pThis = (wtp_t*) arg; + wti_t *pWti = (wti_t*) arg; + wtp_t *pThis; BEGINfunc + ISOBJ_TYPE_assert(pWti, wti); + pThis = pWti->pWtp; ISOBJ_TYPE_assert(pThis, wtp); - pThis->iCurNumWrkThrd--; - wtpSignalWrkrTermination(pThis); + DBGPRINTF("%s: Worker thread %lx requested to be cancelled.\n", + wtpGetDbgHdr(pThis), (unsigned long) pWti); + + wtpWrkrExecCleanup(pWti); - dbgprintf("%s: thread CANCELED with %d workers running.\n", wtpGetDbgHdr(pThis), pThis->iCurNumWrkThrd); ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ } @@ -421,12 +329,13 @@ wtpWrkrExecCancelCleanup(void *arg) static void * wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in wtp! */ { - DEFiRet; - DEFVARS_mutexProtection; + uchar *pszDbgHdr; + uchar thrdName[32] = "rs:"; wti_t *pWti = (wti_t*) arg; wtp_t *pThis; sigset_t sigSet; + BEGINfunc ISOBJ_TYPE_assert(pWti, wti); pThis = pWti->pWtp; ISOBJ_TYPE_assert(pThis, wtp); @@ -434,41 +343,26 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in sigfillset(&sigSet); pthread_sigmask(SIG_BLOCK, &sigSet, NULL); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - - /* do some late initialization */ - - pthread_cleanup_push(wtpWrkrExecCancelCleanup, pThis); - - /* finally change to RUNNING state. We need to check if we actually should still run, - * because someone may have requested us to shut down even before we got a chance to do - * our init. That would be a bad race... -- rgerhards, 2008-01-16 - */ - wtiSetState(pWti, eWRKTHRD_RUNNING, 0, MUTEX_ALREADY_LOCKED); /* we are running now! */ - - do { - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - iRet = wtiWorker(pWti); /* just to make sure: this is NOT protected by the mutex! */ - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - } while(pThis->iCurNumWrkThrd == 1 && pThis->bInactivityGuard == 1); - /* inactivity guard prevents shutdown of all workers while one should be running due to race - * condition. It can lead to one more worker running than desired, but that is acceptable. After - * all, that worker will shutdown itself due to inactivity timeout. If, however, none were running - * when one was required, processing could come to a halt. -- rgerhards, 2008-01-21 - */ +# if HAVE_PRCTL && defined PR_SET_NAME + /* set thread name - we ignore if the call fails, has no harsh consequences... */ + pszDbgHdr = wtpGetDbgHdr(pThis); + ustrncpy(thrdName+3, pszDbgHdr, 20); + if(prctl(PR_SET_NAME, thrdName, 0, 0, 0) != 0) { + DBGPRINTF("prctl failed, not setting thread name for '%s'\n", wtpGetDbgHdr(pThis)); + } +# endif + pthread_cleanup_push(wtpWrkrExecCancelCleanup, pWti); + wtiWorker(pWti); pthread_cleanup_pop(0); - pThis->iCurNumWrkThrd--; - wtpSignalWrkrTermination(pThis); - - dbgprintf("%s: Worker thread %lx, terminated, num workers now %d\n", - wtpGetDbgHdr(pThis), (unsigned long) pWti, pThis->iCurNumWrkThrd); - - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + wtpWrkrExecCleanup(pWti); ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ pthread_exit(0); } #pragma GCC diagnostic warning "-Wempty-body" @@ -476,27 +370,20 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in /* start a new worker */ static rsRetVal -wtpStartWrkr(wtp_t *pThis, int bLockMutex) +wtpStartWrkr(wtp_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; wti_t *pWti; int i; int iState; + DEFiRet; ISOBJ_TYPE_assert(pThis, wtp); - wtpProcessThrdChanges(pThis); // TODO: Performance: this causes a lot of FUTEX calls - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); + d_pthread_mutex_lock(&pThis->mutWtp); - pThis->iCurNumWrkThrd++; - - /* find free spot in thread table. If we find at least one worker that is in initialization, - * we do NOT start a new one. Let's give the other one a chance, first. - */ + /* find free spot in thread table. */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - if(wtiGetState(pThis->pWrkr[i], LOCK_MUTEX) == eWRKTHRD_STOPPED) { + if(wtiGetState(pThis->pWrkr[i]) == WRKTHRD_STOPPED) { break; } } @@ -504,25 +391,20 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) if(i == pThis->iNumWorkerThreads) ABORT_FINALIZE(RS_RET_NO_MORE_THREADS); + if(i == 0 || pThis->toWrkShutdown == -1) { + wtiSetAlwaysRunning(pThis->pWrkr[i]); + } + pWti = pThis->pWrkr[i]; - wtiSetState(pWti, eWRKTHRD_RUN_CREATED, 0, LOCK_MUTEX); - iState = pthread_create(&(pWti->thrdID), NULL, wtpWorker, (void*) pWti); - dbgprintf("%s: started with state %d, num workers now %d\n", - wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd); - - /* we try to give the starting worker a little boost. It won't help much as we still - * hold the queue's mutex, but at least it has a chance to start on a single-CPU system. - */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); -# endif + wtiSetState(pWti, WRKTHRD_RUNNING); + iState = pthread_create(&(pWti->thrdID), &pThis->attrThrd, wtpWorker, (void*) pWti); + ATOMIC_INC(pThis->iCurNumWrkThrd); /* we got one more! */ - /* indicate we just started a worker and would like to see it running */ - wtpSetInactivityGuard(pThis, 1, MUTEX_ALREADY_LOCKED); + DBGPRINTF("%s: started with state %d, num workers now %d\n", + wtpGetDbgHdr(pThis), iState, ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd)); finalize_it: - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + d_pthread_mutex_unlock(&pThis->mutWtp); RETiRet; } @@ -539,7 +421,6 @@ rsRetVal wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) { DEFiRet; - DEFVARS_mutexProtection; int nMissing; /* number workers missing to run */ int i; @@ -548,29 +429,24 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) if(nMaxWrkr == 0) FINALIZE; - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - if(nMaxWrkr > pThis->iNumWorkerThreads) /* limit to configured maximum */ nMaxWrkr = pThis->iNumWorkerThreads; - nMissing = nMaxWrkr - pThis->iCurNumWrkThrd; + nMissing = nMaxWrkr - ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd); if(nMissing > 0) { - dbgprintf("%s: high activity - starting %d additional worker thread(s).\n", wtpGetDbgHdr(pThis), nMissing); + DBGPRINTF("%s: high activity - starting %d additional worker thread(s).\n", + wtpGetDbgHdr(pThis), nMissing); /* start the rqtd nbr of workers */ for(i = 0 ; i < nMissing ; ++i) { - CHKiRet(wtpStartWrkr(pThis, MUTEX_ALREADY_LOCKED)); - } - } else { - if(nMaxWrkr > 0) { - dbgprintf("wtpAdviseMaxWorkers signals busy\n"); - wtpWakeupWrkr(pThis); + CHKiRet(wtpStartWrkr(pThis)); } + } else { + pthread_cond_signal(pThis->pcondBusy); } finalize_it: - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); RETiRet; } @@ -584,35 +460,9 @@ DEFpropSetMethPTR(wtp, pmutUsr, pthread_mutex_t) DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t) DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)) -DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)) -DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) -DEFpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*, void*)) -DEFpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)) - - -/* return the current number of worker threads. - * TODO: atomic operation would bring a nice performance - * enhancemcent - * rgerhards, 2008-01-27 - */ -int -wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex) -{ - DEFVARS_mutexProtection; - int iNumWrkr; - - BEGINfunc - ISOBJ_TYPE_assert(pThis, wtp); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - iNumWrkr = pThis->iCurNumWrkThrd; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - ENDfunc - return iNumWrkr; -} +DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)) +DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)) +DEFpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)) /* set the debug header message @@ -636,7 +486,7 @@ wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg) pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ @@ -666,6 +516,5 @@ BEGINObjClassInit(wtp, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wtp) -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/wtp.h b/runtime/wtp.h index b9cb07c5..05c02a8c 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -27,18 +27,9 @@ #include <pthread.h> #include "obj.h" -/* commands and states for worker threads. */ -typedef enum { - eWRKTHRD_STOPPED = 0, /* worker thread is not running (either actually never ran or was shut down) */ - eWRKTHRD_TERMINATING = 1,/* worker thread has shut down, but some finalzing is still needed */ - /* ALL active states MUST be numerically higher than eWRKTHRD_TERMINATED and NONE must be lower! */ - eWRKTHRD_RUN_CREATED = 2,/* worker thread has been created, but not yet begun initialization (prob. not yet scheduled) */ - eWRKTHRD_RUN_INIT = 3, /* worker thread is initializing, but not yet fully running */ - eWRKTHRD_RUNNING = 4, /* worker thread is up and running and shall continue to do so */ - eWRKTHRD_SHUTDOWN = 5, /* worker thread is running but shall terminate when wtp is empty */ - eWRKTHRD_SHUTDOWN_IMMEDIATE = 6/* worker thread is running but shall terminate even if wtp is full */ - /* SHUTDOWN_IMMEDIATE MUST alsways be the numerically highest state! */ -} qWrkCmd_t; +/* states for worker threads. */ +#define WRKTHRD_STOPPED FALSE +#define WRKTHRD_RUNNING TRUE /* possible states of a worker thread pool */ @@ -50,37 +41,31 @@ typedef enum { /* the worker thread pool (wtp) object */ -typedef struct wtp_s { +struct wtp_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ wtpState_t wtpState; int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ struct wti_s **pWrkr;/* array with control structure for the worker thread(s) associated with this wtp */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ - int bInactivityGuard;/* prevents inactivity due to race condition */ rsRetVal (*pConsumer)(void *); /* user-supplied consumer function for dewtpd messages */ /* synchronization variables */ - pthread_mutex_t mutThrdShutdwn; /* mutex to guard thread shutdown processing */ - pthread_mutex_t mut; /* mutex for the wtp's thread management */ + pthread_mutex_t mutWtp; /* mutex for the wtp's thread management */ pthread_cond_t condThrdTrm;/* signalled when threads terminate */ - int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* user objects */ - void *pUsr; /* pointer to user object */ + void *pUsr; /* pointer to user object (in this case, the queue the wtp belongs to) */ + pthread_attr_t attrThrd;/* attribute for new threads (created just once and cached here) */ pthread_mutex_t *pmutUsr; pthread_cond_t *pcondBusy; /* condition the user will signal "busy again, keep runing" on (awakes worker) */ rsRetVal (*pfChkStopWrkr)(void *pUsr, int); + rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */ + rsRetVal (*pfObjProcessed)(void *pUsr, wti_t *pWti); /* indicate user object is processed */ rsRetVal (*pfRateLimiter)(void *pUsr); - rsRetVal (*pfIsIdle)(void *pUsr, int); - rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int); - rsRetVal (*pfOnIdle)(void *pUsr, int); - rsRetVal (*pfOnWorkerCancel)(void *pUsr, void*pWti); - rsRetVal (*pfOnWorkerStartup)(void *pUsr); - rsRetVal (*pfOnWorkerShutdown)(void *pUsr); + rsRetVal (*pfDoWork)(void *pUsr, void *pWti); /* end user objects */ uchar *pszDbgHdr; /* header string for debug messages */ -} wtp_t; +}; /* some symbolic constants for easier reference */ @@ -91,25 +76,18 @@ rsRetVal wtpConstructFinalize(wtp_t *pThis); rsRetVal wtpDestruct(wtp_t **ppThis); rsRetVal wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr); rsRetVal wtpProcessThrdChanges(wtp_t *pThis); -rsRetVal wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex); -rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex); +rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex); rsRetVal wtpSetState(wtp_t *pThis, wtpState_t iNewState); -rsRetVal wtpWakeupWrkr(wtp_t *pThis); rsRetVal wtpWakeupAllWrkr(wtp_t *pThis); rsRetVal wtpCancelAll(wtp_t *pThis); rsRetVal wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg); -rsRetVal wtpSignalWrkrTermination(wtp_t *pWtp); rsRetVal wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout); -int wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex); PROTOTYPEObjClassInit(wtp); PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*,void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)); +PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)); +PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)); +PROTOTYPEpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)); PROTOTYPEpropSetMeth(wtp, toWrkShutdown, long); PROTOTYPEpropSetMeth(wtp, wtpState, wtpState_t); PROTOTYPEpropSetMeth(wtp, iMaxWorkerThreads, int); diff --git a/runtime/zlibw.c b/runtime/zlibw.c new file mode 100644 index 00000000..2b386213 --- /dev/null +++ b/runtime/zlibw.c @@ -0,0 +1,125 @@ +/* The zlibwrap object. + * + * This is an rsyslog object wrapper around zlib. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <string.h> +#include <assert.h> +#include <zlib.h> + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "zlibw.h" + +MODULE_TYPE_LIB + +/* static data */ +DEFobjStaticHelpers + + +/* ------------------------------ methods ------------------------------ */ + +/* zlib make strong use of macros for its interface functions, so we can not simply + * pass function pointers to them. Instead, we create very small wrappers which call + * the relevant entry points. + */ + +static int myDeflateInit(z_streamp strm, int level) +{ + return deflateInit(strm, level); +} + +static int myDeflateInit2(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy) +{ + return deflateInit2(strm, level, method, windowBits, memLevel, strategy); +} + +static int myDeflateEnd(z_streamp strm) +{ + return deflateEnd(strm); +} + +static int myDeflate(z_streamp strm, int flush) +{ + return deflate(strm, flush); +} + + +/* queryInterface function + * rgerhards, 2008-03-05 + */ +BEGINobjQueryInterface(zlibw) +CODESTARTobjQueryInterface(zlibw) + if(pIf->ifVersion != zlibwCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DeflateInit = myDeflateInit; + pIf->DeflateInit2 = myDeflateInit2; + pIf->Deflate = myDeflate; + pIf->DeflateEnd = myDeflateEnd; +finalize_it: +ENDobjQueryInterface(zlibw) + + +/* Initialize the zlibw class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINAbstractObjClassInit(zlibw, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ + /* request objects we use */ + + /* set our own handlers */ +ENDObjClassInit(zlibw) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + CHKiRet(zlibwClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + /* Initialize all classes that are in our module - this includes ourselfs */ +ENDmodInit +/* vi:set ai: + */ diff --git a/runtime/zlibw.h b/runtime/zlibw.h new file mode 100644 index 00000000..63d8f386 --- /dev/null +++ b/runtime/zlibw.h @@ -0,0 +1,46 @@ +/* The zlibw object. It encapsulates the zlib functionality. The primary + * purpose of this wrapper class is to enable rsyslogd core to be build without + * zlib libraries. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_ZLIBW_H +#define INCLUDED_ZLIBW_H + +#include <zlib.h> + +/* interfaces */ +BEGINinterface(zlibw) /* name must also be changed in ENDinterface macro! */ + int (*DeflateInit)(z_streamp strm, int); + int (*DeflateInit2)(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy); + int (*Deflate)(z_streamp strm, int); + int (*DeflateEnd)(z_streamp strm); +ENDinterface(zlibw) +#define zlibwCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(zlibw); + +/* the name of our library binary */ +#define LM_ZLIBW_FILENAME "lmzlibw" + +#endif /* #ifndef INCLUDED_ZLIBW_H */ |