From 65f74a712b111f946faaddd0583a6c7b22bd062f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Oct 2008 14:25:23 +0200 Subject: added $UDPServerTimeRequery option ...which enables to work with less acurate timestamps in favor of performance. This enables querying of the time only every n-th time if imudp is running in the tight receive loop (aka receiving messsages at a high rate) --- runtime/msg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 21cb2c64..d3c0718b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -119,6 +119,7 @@ struct msg { PROTOTYPEObjClassInit(msg); char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); @@ -180,6 +181,5 @@ extern void (*funcMsgPrepareEnqueue)(msg_t *pMsg); #define MsgPrepareEnqueue(pMsg) funcMsgPrepareEnqueue(pMsg) #endif /* #ifndef MSG_H_INCLUDED */ -/* - * vi:set ai: +/* vim:set ai: */ -- cgit v1.2.3 From 8528344ef58b5d2907bba8809f63d0bca2ce8d38 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 14:26:41 +0200 Subject: "output" timestamp now taken from mesg's time generated This enhances performance and, as some have pointed out, is probably also more consistent with what users expect how the various output-timestamp related function should work. This commit needs some more testing. --- runtime/msg.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index d3c0718b..d2fc2f30 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -99,6 +99,13 @@ struct msg { cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. + While this field looks redundant, it is required because a Unix timestamp + is used at later processing stages (namely in the output arena). Thanks to + the subleties of how time is defined, there is no reliable way to reconstruct + the Unix timestamp from the syslogTime fields (in practice, we may be close + enough to reliable, but I prefer to leave the subtle things to the OS, where + it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ @@ -119,7 +126,7 @@ struct msg { PROTOTYPEObjClassInit(msg); char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); -rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime); +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); -- cgit v1.2.3 From ace4f2f75202aec39449dac11b9eb1deca7428d7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Oct 2008 18:55:11 +0200 Subject: reordered imudp processing. Message parsing is now done as part of main message queue worker processing (was part of the input thread) This should also improve performance, as potentially more work is done in parallel. --- runtime/msg.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index d2fc2f30..98635f85 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -52,7 +52,10 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; pthread_mutex_t mut; - int iRefCount; /* reference counter (0 = unused) */ + flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because + once data has entered the queue, this property is no longer needed. */ + short iRefCount; /* reference counter (0 = unused) */ + short bIsParsed; /* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */ short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -60,8 +63,6 @@ struct msg { * sockets. All in all, the parser would need parse templates, that would * resolve all these issues... rgerhards, 2005-10-06 */ - flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because - once data has entered the queue, this property is no longer needed. */ short iSeverity; /* the severity 0..7 */ uchar *pszSeverity; /* severity as string... */ int iLenSeverity; /* ... and its length. */ -- cgit v1.2.3 From 6c6e9a0f3f7d454ba9553a750b195d7f99c7299a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 13:45:56 +0200 Subject: moved bParseHostname and bIsParsed to msgFlags This enables us to use more efficient calling conventions and also helps us keep the on-disk structure of a msg object more consistent in future releases. --- runtime/msg.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 98635f85..d98111a8 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -55,7 +55,6 @@ struct msg { flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ - short bIsParsed; /* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */ short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -122,6 +121,18 @@ struct msg { int msgFlags; /* flags associated with this message */ }; + +/* message flags (msgFlags), not an enum for historical reasons + */ +#define NOFLAG 0x000 /* no flag is set (to be used when a flag must be specified and none is required) */ +#define INTERNAL_MSG 0x001 /* msg generated by logmsgInternal() --> special handling */ +/* 0x002 not used because it was previously a known value - rgerhards, 2008-10-09 */ +#define IGNDATE 0x004 /* ignore, if given, date in message and use date of reception as msg date */ +#define MARK 0x008 /* this message is a mark */ +#define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ +#define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ + + /* function prototypes */ PROTOTYPEObjClassInit(msg); -- cgit v1.2.3 From ead2c355e3261f98817ccd52bc3644103140e824 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 26 Jan 2009 22:30:55 +0100 Subject: bugfix: unitialized mutex was used in msg.c:getPRI This was subtle, because getPRI is called as part of the debugging code (always executed) in syslogd.c:logmsg. --- runtime/msg.h | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index d98111a8..c8350626 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,6 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; +short bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ -- cgit v1.2.3 From aba90e82484118f3568ec51c01de5ba845da589a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 May 2009 17:06:52 +0200 Subject: added capability to run multiple tcp listeners (on different ports) Well, actually this and a lot of related things. I improved the testbench so that the new capabilities are automatically tested and also did some general cleanup. The current multiple tcp listener solution will probably receive some further cleanup, too, but looks quite OK so far. I also reviewed the way tcpsrv et all work, in preparation of using this code for imdiag. I need to document the findings, especially as the code is rather complicated "thanks" to the combination of plain tcp and gssapi transport modes. --- runtime/msg.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index c8350626..a14f6b15 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -158,7 +158,7 @@ char *getSeverity(msg_t *pM); char *getSeverityStr(msg_t *pM); char *getFacility(msg_t *pM); char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, char*); +void MsgSetInputName(msg_t *pMsg, uchar*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); @@ -171,15 +171,15 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); char *getTAG(msg_t *pM); int getHOSTNAMELen(msg_t *pM); char *getHOSTNAME(msg_t *pM); -char *getRcvFrom(msg_t *pM); +uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); char *getStructuredData(msg_t *pM); int getProgramNameLen(msg_t *pM); char *getProgramName(msg_t *pM); -void MsgSetRcvFrom(msg_t *pMsg, char* pszRcvFrom); +void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); -void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME); +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); -- cgit v1.2.3 From ca0ddc30a3edce02a440904a01f0b866c0f82b5a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 15:31:08 +0200 Subject: completed multi-ruleset core support ... as well as added multi-ruleset support for imtcp --- runtime/msg.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..b42f641f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -120,6 +120,7 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ }; @@ -167,6 +168,7 @@ char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); char *getTAG(msg_t *pM); int getHOSTNAMELen(msg_t *pM); -- cgit v1.2.3 From 015d17ca70e81ad998e32cdfeed3cd925fd7dedc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 08:46:45 +0200 Subject: some performance optimizations - saved gettimeofday() calls in imtcp (and increased reception buffer) - somewhat optimized stringbuf.c - some other optimizations --- runtime/msg.h | 48 +++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..1689bbbc 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -120,6 +120,9 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ + /* now follow fixed-size buffers to safe some time otherwise used for allocs */ + uchar bufPRI[5]; + }; @@ -137,45 +140,20 @@ short bDoLock; /* use the mutex? */ /* function prototypes */ PROTOTYPEObjClassInit(msg); -char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -int getProtocolVersion(msg_t *pM); -char *getProtocolVersionString(msg_t *pM); -int getMSGLen(msg_t *pM); -char *getRawMsg(msg_t *pM); -char *getUxTradMsg(msg_t *pM); -char *getMSG(msg_t *pM); -char *getPRI(msg_t *pM); -int getPRIi(msg_t *pM); -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt); -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt); -char *getSeverity(msg_t *pM); -char *getSeverityStr(msg_t *pM); -char *getFacility(msg_t *pM); -char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, uchar*); +void MsgSetInputName(msg_t *pMsg, uchar*, size_t); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); -char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); -int getPROCIDLen(msg_t *pM); -char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, char* pszTAG); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); -char *getTAG(msg_t *pM); -int getHOSTNAMELen(msg_t *pM); -char *getHOSTNAME(msg_t *pM); -uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -char *getStructuredData(msg_t *pM); -int getProgramNameLen(msg_t *pM); -char *getProgramName(msg_t *pM); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); @@ -184,13 +162,29 @@ int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); -char *getMSGID(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +/* TODO: remove these five (so far used in action.c) */ +char *getMSG(msg_t *pM); +char *getHOSTNAME(msg_t *pM); +char *getPROCID(msg_t *pM); +char *getAPPNAME(msg_t *pM); +int getMSGLen(msg_t *pM); + +char *getHOSTNAME(msg_t *pM); +int getHOSTNAMELen(msg_t *pM); +char *getProgramName(msg_t *pM); +int getProgramNameLen(msg_t *pM); +uchar *getRcvFrom(msg_t *pM); + +#if 0 +char *getUxTradMsg(msg_t *pM); +#endif + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains * us quite some performance and is fully abstracted using this header file. -- cgit v1.2.3 From f7579e68a67364c8040966be57c2eae4c9550ee5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 11:36:05 +0200 Subject: done various optimizations to the stringbuf and its users --- runtime/msg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 1689bbbc..20360641 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -73,7 +73,8 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - uchar *pszPRI; /* the PRI as a string */ + //uchar *pszPRI; /* the PRI as a string */ + uchar bufPRI[5]; int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we @@ -121,7 +122,6 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ /* now follow fixed-size buffers to safe some time otherwise used for allocs */ - uchar bufPRI[5]; }; -- cgit v1.2.3 From 74b2b24f508be90d20961304d5e3cce648f3eb7c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 12:31:27 +0200 Subject: removed long-obsoleted property UxTradMsg ... as well as some cleanup (some commented-out code is left to support UxTradMsg again is someone really complains ;)). --- runtime/msg.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 20360641..fe9f87fa 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,7 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; -short bDoLock; /* use the mutex? */ + bool bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ @@ -73,13 +73,13 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - //uchar *pszPRI; /* the PRI as a string */ - uchar bufPRI[5]; + uchar bufPRI[5]; /* PRI as string */ int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we * need to preserve cryptographic verifiers. */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ int iLenRawMsg; /* length of raw message */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ @@ -158,7 +158,7 @@ void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); +rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); @@ -183,6 +183,7 @@ uchar *getRcvFrom(msg_t *pM); #if 0 char *getUxTradMsg(msg_t *pM); +int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); #endif /* The MsgPrepareEnqueue() function is a macro for performance reasons. -- cgit v1.2.3 From 56e462610db0dc71cfc2e4af17d1eb27bd67fae7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 12:56:58 +0200 Subject: further optimized message object pri, facility and severity string generation simplified --- runtime/msg.h | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 74ff9e60..a3058349 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,12 +51,12 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; - bool bDoLock; /* use the mutex? */ + bool bDoLock; /* use the mutex? */ + bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ - short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, * it is available when we have a forwarder (e.g. rfc3195d) using local @@ -66,15 +66,9 @@ struct msg { short iSeverity; /* the severity 0..7 */ uchar *pszSeverity; /* severity as string... */ int iLenSeverity; /* ... and its length. */ - uchar *pszSeverityStr; /* severity name... */ - int iLenSeverityStr; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ uchar *pszFacility; /* Facility as string... */ int iLenFacility; /* ... and its length. */ - uchar *pszFacilityStr; /* facility name... */ - int iLenFacilityStr; /* ... and its length. */ - uchar bufPRI[5]; /* PRI as string */ - int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we * need to preserve cryptographic verifiers. @@ -83,8 +77,6 @@ struct msg { int iLenRawMsg; /* length of raw message */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszUxTradMsg; /* the traditional UNIX message */ - int iLenUxTradMsg;/* Length of the traditional UNIX message */ uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ @@ -122,7 +114,6 @@ struct msg { char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ - /* now follow fixed-size buffers to safe some time otherwise used for allocs */ }; @@ -182,11 +173,6 @@ char *getProgramName(msg_t *pM); int getProgramNameLen(msg_t *pM); uchar *getRcvFrom(msg_t *pM); -#if 0 -char *getUxTradMsg(msg_t *pM); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); -#endif - /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains * us quite some performance and is fully abstracted using this header file. -- cgit v1.2.3 From 34b60b389e463e260f880ca3efc591b4db2cd18a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 13:33:57 +0200 Subject: some more optimization in msg object string handling --- runtime/msg.h | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index a3058349..703fdd9f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -50,12 +50,12 @@ */ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because + once data has entered the queue, this property is no longer needed. */ pthread_mutexattr_t mutAttr; + pthread_mutex_t mut; bool bDoLock; /* use the mutex? */ bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ - pthread_mutex_t mut; - flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because - once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -64,17 +64,11 @@ struct msg { * resolve all these issues... rgerhards, 2005-10-06 */ short iSeverity; /* the severity 0..7 */ - uchar *pszSeverity; /* severity as string... */ - int iLenSeverity; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ - uchar *pszFacility; /* Facility as string... */ - int iLenFacility; /* ... and its length. */ - uchar *pszRawMsg; /* message as it was received on the - * wire. This is important in case we - * need to preserve cryptographic verifiers. - */ - short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ + uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we + * need to preserve cryptographic verifiers. */ int iLenRawMsg; /* length of raw message */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ uchar *pszTAG; /* pointer to tag value */ @@ -114,6 +108,9 @@ struct msg { char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ + /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + char pszTimestamp3164[16]; + char pszTimestamp3339[33]; }; -- cgit v1.2.3 From 8628312396b1535c41124e499d292f4d1e77d955 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 13:22:21 +0200 Subject: cleaned up/optimized raw message handling in msg object --- runtime/msg.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 703fdd9f..f2701780 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -28,6 +28,9 @@ #ifndef MSG_H_INCLUDED #define MSG_H_INCLUDED 1 +/* some configuration constants */ +#define CONF_RAWMSG_BUFSIZE 101 + #include #include "obj.h" #include "syslogd-types.h" @@ -109,6 +112,7 @@ struct msg { int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ char pszTimestamp3164[16]; char pszTimestamp3339[33]; }; @@ -149,7 +153,8 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSG(msg_t *pMsg, char* pszMSG); -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, unsigned short *pbMustBeFreed); -- cgit v1.2.3 From 2f86678c577ee469852ffae35123c4a90b12d214 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 14:30:21 +0200 Subject: optimized TAG handling in msg object --- runtime/msg.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index f2701780..13c2c4ea 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -30,6 +30,7 @@ /* some configuration constants */ #define CONF_RAWMSG_BUFSIZE 101 +#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ #include #include "obj.h" @@ -74,7 +75,6 @@ struct msg { short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ int iLenHOSTNAME; /* Length of HOSTNAME */ @@ -113,6 +113,10 @@ struct msg { ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ + union { + uchar *pszTAG; /* pointer to tag value */ + uchar szBuf[CONF_TAG_BUFSIZE]; + } TAG; char pszTimestamp3164[16]; char pszTimestamp3339[33]; }; @@ -143,7 +147,7 @@ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); -void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -- cgit v1.2.3 From df9012f755a305ef48037f10fcc9413406894e66 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 16:20:06 +0200 Subject: slight optimization of template generation --- runtime/msg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 13c2c4ea..48c1090e 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -161,7 +161,7 @@ void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed); + cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); -- cgit v1.2.3 From 2de4964affabc1ccf61bc72426a468fc871a54d0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 16:51:40 +0200 Subject: optimized handling of MSG part in msg object WARNING: currently, message repeation processing is disabled, must be reenabled (but prefer to do some other tests first) --- runtime/msg.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 48c1090e..4cdb0762 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -73,7 +73,8 @@ struct msg { * need to preserve cryptographic verifiers. */ int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ - uchar *pszMSG; /* the MSG part itself */ + short offMSG; /* offset at which the MSG part starts in pszRawMsg */ + //uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ @@ -156,7 +157,7 @@ rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); -void MsgSetMSG(msg_t *pMsg, char* pszMSG); +void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); -- cgit v1.2.3 From f33dd51c802a8d49839aa73fb9167d8bc31ea912 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 17:48:11 +0200 Subject: fixed abort condition with oversize tags this was a regression I introduced this afternoon --- runtime/msg.h | 1 - 1 file changed, 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 4cdb0762..ec18b29d 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -74,7 +74,6 @@ struct msg { int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ - //uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ -- cgit v1.2.3 From 221dc8a3224dcb59a7dd3158716a8d24cee71618 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 18:19:10 +0200 Subject: some more optimizations of the msg_t object (minor) --- runtime/msg.h | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index ec18b29d..34983704 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -56,7 +56,6 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ - pthread_mutexattr_t mutAttr; pthread_mutex_t mut; bool bDoLock; /* use the mutex? */ bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ @@ -69,27 +68,37 @@ struct msg { */ short iSeverity; /* the severity 0..7 */ short iFacility; /* Facility code 0 .. 23*/ - uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we - * need to preserve cryptographic verifiers. */ - int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ + short iLenInputName; /* Length of pszInputName */ + short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + int msgFlags; /* flags associated with this message */ + int iLenRawMsg; /* length of raw message */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ - uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ int iLenHOSTNAME; /* Length of HOSTNAME */ - uchar *pszRcvFrom; /* System message was received from */ int iLenRcvFrom; /* Length of pszRcvFrom */ - uchar *pszRcvFromIP; /* IP of system message was received from */ int iLenRcvFromIP; /* Length of pszRcvFromIP */ + uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we + * need to preserve cryptographic verifiers. */ + uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ + uchar *pszRcvFrom; /* System message was received from */ + uchar *pszRcvFromIP; /* IP of system message was received from */ uchar *pszInputName; /* name of the input module that submitted this message */ - int iLenInputName; /* Length of pszInputName */ - short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ + char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ + char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ + char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ + char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ + char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ + char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ + char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ cstr_t *pCSProgName; /* the (BSD) program name */ cstr_t *pCSStrucData; /* STRUCTURED-DATA */ cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp is used at later processing stages (namely in the output arena). Thanks to @@ -98,19 +107,7 @@ struct msg { enough to reliable, but I prefer to leave the subtle things to the OS, where it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ - char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ - char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ - char *pszRcvdAt_SecFrac;/* time just as fractional seconds (6 charcters) */ - char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ - char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ - char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ - char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ - char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ - char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ - char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ - int msgFlags; /* flags associated with this message */ - ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ union { @@ -119,6 +116,8 @@ struct msg { } TAG; char pszTimestamp3164[16]; char pszTimestamp3339[33]; + char pszTIMESTAMP_SecFrac[7]; /* Note: a pointer is 64 bits/8 char, so this is actually fewer than a pointer! */ + char pszRcvdAt_SecFrac[7]; /* same as above. Both are fractional seconds for their respective timestamp */ }; -- cgit v1.2.3 From ce5869f7c41c8db943d8cbe804b69af40d43e2e6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 18:52:30 +0200 Subject: optimized processing of TAG message field --- runtime/msg.h | 1 - 1 file changed, 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 34983704..0f9bd95e 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -145,7 +145,6 @@ void MsgSetInputName(msg_t *pMsg, uchar*, size_t); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); -void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); -- cgit v1.2.3 From d798f5b66f4ae699f352b6c40abd07495eff8f94 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 11:22:07 +0200 Subject: some more optimizations - done malloc() instead of calloc() for msg_t, as we have large space which needs not be initialized - shrunk syslogTime structure in the hope to get better cache and write performance (non-aligned data should not hurt much here) --- runtime/msg.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 0f9bd95e..9113882a 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,6 +51,11 @@ * will be decremented. If it is 1, however, the object is actually * destroyed. To make this work, it is vital that MsgAddRef() is * called each time a "copy" is stored somewhere. + * + * WARNING: this structure is not calloc()ed, so be careful when + * adding new fields. You need to initialize them in + * msgBaseConstruct(). That function header comment also describes + * why this is the case. */ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ -- cgit v1.2.3 From b50d13a6a97c0b6fa14807775ae0edf52ef015fb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 14:50:03 +0200 Subject: restored repeated message reduction processing --- runtime/msg.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 9113882a..c38cb788 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -28,10 +28,6 @@ #ifndef MSG_H_INCLUDED #define MSG_H_INCLUDED 1 -/* some configuration constants */ -#define CONF_RAWMSG_BUFSIZE 101 -#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ - #include #include "obj.h" #include "syslogd-types.h" @@ -162,6 +158,7 @@ rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); +rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); -- cgit v1.2.3 From 86e37f70fe0e9de0e00362990c73536843c8fef3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 16:14:19 +0200 Subject: more strict parsing of the hostname in rfc3164 mode ... hopefully removes false positives (but may cause some trouble with hostname parsing). For details, see this bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=126 This patch is not optimal for v4 - another one will follow. The spirit of this commit is to enable easier backporting if someone is interested in doing so. --- runtime/msg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index c38cb788..d70b939a 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -152,7 +152,7 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); -void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); +//void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); -- cgit v1.2.3 From 662ad3e4bf8dbd317d18aa1afcbf3e8b9e424506 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 16:32:29 +0200 Subject: optimized hostname processing --- runtime/msg.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index d70b939a..fd4d650b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -3,7 +3,7 @@ * * File begun on 2007-07-13 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -152,8 +152,7 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); -//void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); -- cgit v1.2.3 From b2fa740b9ab5fb9e85309b3307f3fca21f625ab1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 17:14:42 +0200 Subject: optimized TAG handling --- runtime/msg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index fd4d650b..4bfc1e3f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -111,6 +111,7 @@ struct msg { struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ + uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE]; union { uchar *pszTAG; /* pointer to tag value */ uchar szBuf[CONF_TAG_BUFSIZE]; @@ -158,7 +159,6 @@ void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); -void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); -- cgit v1.2.3 From d88803c8e471b0b822108106137f5383c7efd57e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 26 Jun 2009 13:42:52 +0200 Subject: got rid of the recursive requirement for msg_t mutex (finally!) --- runtime/msg.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 4bfc1e3f..56ba5b80 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -168,14 +168,14 @@ rsRetVal MsgEnableThreadSafety(void); /* TODO: remove these five (so far used in action.c) */ char *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); -char *getPROCID(msg_t *pM); -char *getAPPNAME(msg_t *pM); +char *getPROCID(msg_t *pM, bool bLockMutex); +char *getAPPNAME(msg_t *pM, bool bLockMutex); int getMSGLen(msg_t *pM); char *getHOSTNAME(msg_t *pM); int getHOSTNAMELen(msg_t *pM); -char *getProgramName(msg_t *pM); -int getProgramNameLen(msg_t *pM); +char *getProgramName(msg_t *pM, bool bLockMutex); +int getProgramNameLen(msg_t *pM, bool bLockMutex); uchar *getRcvFrom(msg_t *pM); /* The MsgPrepareEnqueue() function is a macro for performance reasons. -- cgit v1.2.3 From 7ea63db2f9d6b375dc95696e47357d73927f2d3d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 26 Jun 2009 15:31:40 +0200 Subject: optimization: propert names are now internally identified by integers --- runtime/msg.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 56ba5b80..e6b25e6c 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -33,6 +33,7 @@ #include "syslogd-types.h" #include "template.h" + /* rgerhards 2004-11-08: The following structure represents a * syslog message. * @@ -160,7 +161,7 @@ void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); + propid_t propID, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); @@ -177,6 +178,8 @@ int getHOSTNAMELen(msg_t *pM); char *getProgramName(msg_t *pM, bool bLockMutex); int getProgramNameLen(msg_t *pM, bool bLockMutex); uchar *getRcvFrom(msg_t *pM); +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains -- cgit v1.2.3 From de84a12f8a5f140c0f7b8e00f4cac92ef13cd866 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 29 Jun 2009 16:53:26 +0200 Subject: introduced the idea of detached properties some things inside the message can be used over a large number of messages and need to to be allocated and re-written every time. I now begin to implement this as a "prop_t" object, first use for the inputName. Some input modules are already converted, some others to go. Will do a little performance check on the new method before I go further. Also, this commit has some cleanup and a few bug fixes that prevented compiliation in debug mode (I overlooked this as I did not compile for debug, what I normally do, and the automatted test also does not do that) --- runtime/msg.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index e6b25e6c..a1fc535b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -72,7 +72,6 @@ struct msg { short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ - short iLenInputName; /* Length of pszInputName */ short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ int msgFlags; /* flags associated with this message */ int iLenRawMsg; /* length of raw message */ @@ -86,7 +85,6 @@ struct msg { uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ uchar *pszRcvFrom; /* System message was received from */ uchar *pszRcvFromIP; /* IP of system message was received from */ - uchar *pszInputName; /* name of the input module that submitted this message */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ @@ -100,6 +98,7 @@ struct msg { cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + prop_t *pInputName; /* input name property */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -144,7 +143,7 @@ rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -void MsgSetInputName(msg_t *pMsg, uchar*, size_t); +void MsgSetInputName(msg_t *pMsg, prop_t*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); @@ -166,6 +165,12 @@ char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); + +// REMOVE: +void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len); + + + /* TODO: remove these five (so far used in action.c) */ char *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); -- cgit v1.2.3 From e397c34d2a6c7c1e4c116fd2363cb173e32eb2a2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 30 Jun 2009 15:21:15 +0200 Subject: finished transition to using inputName property ... plus some celanup and adding minor missing functionality (the rule debug info again tell the property name, not just number). --- runtime/msg.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index a1fc535b..7f84da35 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -166,11 +166,6 @@ rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); -// REMOVE: -void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len); - - - /* TODO: remove these five (so far used in action.c) */ char *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); @@ -184,6 +179,7 @@ char *getProgramName(msg_t *pM, bool bLockMutex); int getProgramNameLen(msg_t *pM, bool bLockMutex); uchar *getRcvFrom(msg_t *pM); rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); +uchar *propIDToName(propid_t propID); /* The MsgPrepareEnqueue() function is a macro for performance reasons. -- cgit v1.2.3 From aaffc4281e0b26f419a3fc341461f2fc479080b8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 30 Jun 2009 18:45:41 +0200 Subject: introduced a new way of handling the RcvFrom property ... plus a fix for a long-time bug in obj-types.h. That lead to the object pointer only then to become NULL when the object was actually destructed, I discovered this issue during introduction of the pRcvFrom property in msg_t, but it potentially had other effects, too. I am not sure if some experienced instability resulted from this bug OR if its fix will cause harm to so-far "correctly" running code. The later may very well be. Thus I will change it only for the current branch and also the beta, but not in all old builds. Let's see how things evolve. --- runtime/msg.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 7f84da35..59046fc1 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -78,12 +78,10 @@ struct msg { int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ int iLenHOSTNAME; /* Length of HOSTNAME */ - int iLenRcvFrom; /* Length of pszRcvFrom */ int iLenRcvFromIP; /* Length of pszRcvFromIP */ uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we * need to preserve cryptographic verifiers. */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ - uchar *pszRcvFrom; /* System message was received from */ uchar *pszRcvFromIP; /* IP of system message was received from */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ @@ -99,6 +97,7 @@ struct msg { cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ prop_t *pInputName; /* input name property */ + prop_t *pRcvFrom; /* name of system message was received from */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -151,7 +150,8 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); +void MsgSetRcvFrom(msg_t *pMsg, prop_t*); +void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); @@ -164,6 +164,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +uchar *getRcvFrom(msg_t *pM); /* TODO: remove these five (so far used in action.c) */ -- cgit v1.2.3 From d6faee67b413d1f257c96a14e46f15ec1868a365 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Jul 2009 13:28:17 +0200 Subject: RcvFromIP now also a property This sets stage to enable use of the property-interface to speed up things (mildly), the next step to be done. I have also fixed one regression of yesterday's changes. --- runtime/msg.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 59046fc1..43f24435 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -78,11 +78,9 @@ struct msg { int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ int iLenHOSTNAME; /* Length of HOSTNAME */ - int iLenRcvFromIP; /* Length of pszRcvFromIP */ uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we * need to preserve cryptographic verifiers. */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ - uchar *pszRcvFromIP; /* IP of system message was received from */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ @@ -98,6 +96,7 @@ struct msg { cstr_t *pCSMSGID; /* MSGID */ prop_t *pInputName; /* input name property */ prop_t *pRcvFrom; /* name of system message was received from */ + prop_t *pRcvFromIP; /* IP of system message was received from */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -152,7 +151,8 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, prop_t*); void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int); -rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); +rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); +rsRetVal MsgSetRcvFromIPStr(msg_t *pMsg, uchar*, int); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); -- cgit v1.2.3 From 7bfa03bdc0b73647ffdbe4b73e5c1649af665fbf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Jul 2009 14:33:19 +0200 Subject: now put the new property-based methods to good use ... hopefully reducing the number of allocs/frees as well as overall memory usage in a busy system (plus that these shared properties hopefully remain in cache longer than its single-instance counterparts...) --- runtime/msg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime/msg.h') diff --git a/runtime/msg.h b/runtime/msg.h index 43f24435..c20fb005 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -150,9 +150,9 @@ void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, prop_t*); -void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int); +void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); -rsRetVal MsgSetRcvFromIPStr(msg_t *pMsg, uchar*, int); +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); -- cgit v1.2.3