diff options
42 files changed, 926 insertions, 184 deletions
@@ -1,6 +1,8 @@ --------------------------------------------------------------------------- Version 5.9.0 [V5-DEVEL] (rgerhards), 2011-03-?? - this begins a new devel branch for v5 +- added new config directive $InputTCPFlowControl to select if tcp + received messages shall be flagged as light delayable or not. - enhanced omhdfs to support batching mode. This permits to increase performance, as we now call the HDFS API with much larger message sizes and far more infrequently @@ -8,12 +10,58 @@ Version 5.9.0 [V5-DEVEL] (rgerhards), 2011-03-?? affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 --------------------------------------------------------------------------- -Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? +Version 5.8.1 [V5-stable] (rgerhards), 2011-04-?? +- bugfix: $myhostname not available in RainerScript (and no error message) + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=233 +- bugfix: doc for impstats had wrong config statements + also, config statements were named a bit inconsistent, resolved that + problem by introducing an alias and only documenting the consistent + statements + Thanks to Marcin for bringing up this problem. +- bugfix: IPv6-address could not be specified in omrelp + this was due to improper parsing of ":" + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=250 +--------------------------------------------------------------------------- +Version 5.8.0 [V5-stable] (rgerhards), 2011-04-12 + +This is the new v5-stable branch, importing all feature from the 5.7.x +versions. To see what has changed in regard to the previous v5-stable, +check the Changelog for 5.7.x below. + +- bugfix: race condition in deferred name resolution + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=238 + Special thanks to Marcin for his persistence in helping to solve this + bug. +- bugfix: DA queue was never shutdown once it was started + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241 +--------------------------------------------------------------------------- +Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-29 +- bugfix: ompgsql did not work properly with ANSI SQL strings + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229 +- bugfix: rsyslog did not build with --disable-regexp configure option + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243 +- bugfix: PRI was invalid on Solaris for message from local log socket +- enhance: added $BOM system property to ease writing byte order masks +- bugfix: RFC5424 parser confused by empty structured data + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237 +- bugfix: error return from strgen caused abort, now causes action to be + ignored (just like a failed filter) +- new sample plugin for a strgen to generate sql statement consumable + by a database plugin +- bugfix: strgen could not be used together with database outputs + because the sql/stdsql option could not be specified. This has been + solved by permitting the strgen to include the opton inside its name. + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195 +--------------------------------------------------------------------------- +Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-16 - improved testbench among others, life tests for ommysql (against a test database) have been added, valgrind-based testing enhanced, ... - enhance: fallback *at runtime* to epoll_create if epoll_create1 is not available. Thanks to Michael Biebl for analysis and patch! +- bugfix: failover did not work correctly if repeated msg reduction was on + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 + affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on - bugfix: minor memory leak in omlibdbi (< 1k per instance and run) - bugfix: (regression) omhdfs did no longer compile - bugfix: omlibdbi did not use password from rsyslog.conf @@ -138,7 +186,7 @@ Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 thanks to Lennart Poettering for this patch * sd-systemd API added as part of rsyslog runtime library --------------------------------------------------------------------------- -Version 5.6.5 [V5-STABLE] (rgerhards), 2011-03-?? +Version 5.6.5 [V5-STABLE] (rgerhards), 2011-03-22 - bugfix: failover did not work correctly if repeated msg reduction was on affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 @@ -759,6 +807,9 @@ Version 4.7.0 [v4-devel] (rgerhards), 2010-04-14 - imported changes from 4.5.6 and below --------------------------------------------------------------------------- Version 4.6.6 [v4-stable] (rgerhards), 2010-11-?? +- bugfix: IPv6-address could not be specified in omrelp + this was due to improper parsing of ":" + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=250 - bugfix: imfile potentially duplicates lines This can happen when 0 bytes are read from the input file, and some writer appends data to the file BEFORE we check if a rollover happens. diff --git a/Makefile.am b/Makefile.am index 53c04922..2d73013b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -112,6 +112,10 @@ if ENABLE_CUST1 SUBDIRS += plugins/cust1 endif +if ENABLE_SMCUSTBINDCDR +SUBDIRS += plugins/sm_cust_bindcdr +endif + if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif @@ -1071,7 +1071,8 @@ prepareBatch(action_t *pAction, batch_t *pBatch) pElem = &(pBatch->pElem[i]); if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) { pElem->state = BATCH_STATE_RDY; - prepareDoActionParams(pAction, pElem); + if(prepareDoActionParams(pAction, pElem) != RS_RET_OK) + pElem->bFilterOK = FALSE; } } RETiRet; diff --git a/configure.ac b/configure.ac index 58b203ff..d6082969 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.8],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.8.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -1081,6 +1081,20 @@ AC_ARG_ENABLE(cust1, AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes) +# A custom strgen that also serves as a sample of how to do +# SQL-generating strgen's +AC_ARG_ENABLE(smcustbindcdr, + [AS_HELP_STRING([--enable-smcustbindcdr],[Compiles smcustbindcdr module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_smcustbindcdr="yes" ;; + no) enable_smcustbindcdr="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-smcustbindcdr) ;; + esac], + [enable_smcustbindcdr=no] +) +AM_CONDITIONAL(ENABLE_SMCUSTBINDCDR, test x$enable_smcustbindcdr = xyes) + + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -1178,6 +1192,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ plugins/omudpspoof/Makefile \ + plugins/sm_cust_bindcdr/Makefile \ plugins/cust1/Makefile \ java/Makefile \ tests/Makefile]) @@ -1223,6 +1238,9 @@ echo " pmcisconames module will be compiled: $enable_pmcisconames" echo " pmaixforwardedfrom module w.be compiled: $enable_pmaixforwardedfrom" echo " pmsnare module will be compiled: $enable_pmsnare" echo +echo "---{ strgen modules }---" +echo " sm_cust_bindcdr module will be compiled: $enable_sm_cust_bindcdr" +echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" echo " libdbi support enabled: $enable_libdbi" diff --git a/doc/impstats.html b/doc/impstats.html index 3b4191e8..cede4874 100644 --- a/doc/impstats.html +++ b/doc/impstats.html @@ -22,16 +22,16 @@ settings, this impact may be severe (for high-load environments). </p> <p><b>Configuration Directives</b>:</p> <ul> -<li>$PStatsInterval <Seconds><br> +<li>$PStatInterval <Seconds><br> Sets the interval, in <b>seconds</b> at which messages are generated. Please note that the actual interval may be a bit longer. We do not try to be precise and so the interval is actually a sleep period which is entered after generating all messages. So the actual interval is what is configured here plus the actual time required to generate messages. In general, the difference should not really matter. -<li>$PStatsFacility <numerical facility><br> +<li>$PStatFacility <numerical facility><br> The numerical syslog facility code to be used for generated messages. Default is 5 (syslog).This is useful for filtering messages.</li> -<li>$PStatsSeverity <numerical severity><br> +<li>$PStatSeverity <numerical severity><br> The numerical syslog severity code to be used for generated messages. Default is 6 (info).This is useful for filtering messages.</li> </ul> @@ -45,8 +45,8 @@ stats may not get turned on in all places.</li> <p>This activates the module and records messages to /var/log/rsyslog-stats in 10 minute intervals:<br> </p> <textarea rows="8" cols="60">$ModLoad impstats -$PStatsInterval 600 -$PStatsSeverity 7 +$PStatInterval 600 +$PStatSeverity 7 syslog.debug /var/log/rsyslog-stats </textarea> diff --git a/doc/imtcp.html b/doc/imtcp.html index 422bbd55..b0aaa3c1 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -57,6 +57,15 @@ instructs imtcp to emit a message if the remote peer closes a connection.<br> after loading imtcp, otherwise it may have no effect.</li> <li><b>$InputTCPServerRun</b> <port><br> Starts a TCP server on selected port</li> +<li><b>$InputTCPFlowControl</b> <<b>on</b>/off><br> +This setting specifies whether some message flow control shall be exercised on the +related TCP input. If set to on, messages are handled as "light delayable", which means +the sender is throttled a bit when the queue becomes near-full. This is done in order +to preserve some queue space for inputs that can not throttle (like UDP), but it +may have some undesired effect in some configurations. Still, we consider this as +a useful setting and thus it is the default. To turn the handling off, simply +configure that explicitely. +</li> <li><b>$InputTCPMaxListeners</b> <number><br> Sets the maximum number of listeners (server ports) supported. Default is 20. This must be set before the first $InputTCPServerRun directive.</li> <li><b>$InputTCPMaxSessions</b> <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> diff --git a/doc/manual.html b/doc/manual.html index 945a5a62..e1f0ebf9 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 5.7.8 (beta branch) of rsyslog.</b> +<p><b>This documentation is for version 5.8.0 (stable branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might @@ -110,5 +110,7 @@ any restriction as long as your license is GPLv3 compatible. If your license is you may even be still permitted to use rsyslog source code. However, then you need to look at the way <a href="licensing.html">rsyslog is licensed</a>.</p> <p>Feedback is always welcome, but if you have a support question, please do not -mail Rainer directly (<a href="free_support.html">why not?</a>). +mail Rainer directly (<a href="free_support.html">why not?</a>) - use the +<a href="http://lists.adiscon.net/mailman/listinfo/rsyslog">rsyslogmailing list</a> +or <a href="http://kb.monitorware.com/rsyslog-f40.html">rsyslog formum</a> instead. </body></html> diff --git a/doc/omrelp.html b/doc/omrelp.html index b3132d78..22e6845f 100644 --- a/doc/omrelp.html +++ b/doc/omrelp.html @@ -44,6 +44,7 @@ special "RSYSLOG_ForwardFormat" (case sensitive!) template is used.<br> # port 2514 *.* :omrelp:centralserv:2514;RSYSLOG_ForwardFormat </textarea> +Note: to use IPv6 addresses, encode them in [::1] format. <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the diff --git a/doc/property_replacer.html b/doc/property_replacer.html index 4d242a34..cd357f67 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -156,6 +156,12 @@ than messages generated somewhere. </td> </tr> <tr> +<td><b>$bom</b></td> +<td>The UTF-8 encoded Unicode byte-order mask (BOM). This may be useful +in templates for RFC5424 support, when the character set is know to be +Unicode.</td> +</tr> +<tr> <td><b>$now</b></td> <td>The current date stamp in the format YYYY-MM-DD</td> </tr> diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 6990c6bd..703e7a6e 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -2,13 +2,16 @@ <html><head><title>rsyslog.conf file</title></head> <body> <h1>rsyslog.conf configuration file</h1> -<p><b>This document is currently being enhanced. Please -pardon its current appearance.</b></p> <p><b>Rsyslogd is configured via the rsyslog.conf file</b>, typically found in /etc. By default, rsyslogd reads the file /etc/rsyslog.conf. This may be changed by a command line option.</p> <p><a href="http://wiki.rsyslog.com/index.php/Configuration_Samples"> -Configuration file examples can be found in the rsyslog wiki</a>.</p> +Configuration file examples can be found in the rsyslog wiki</a>. Also +keep the +<a href="http://www.rsyslog.com/config-snippets/">rsyslog config snippets</a> +on your mind. These are ready-to-use +real building blocks for rsyslog configuration. +</p> <p>There is also one sample file provided together with the documentation set. If you do not like to read, be sure to have at least a quick look at @@ -74,7 +77,7 @@ such features is available in rsyslogd, only.</p> [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008,2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body> diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 7d121a4a..a5d69f1d 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -93,7 +93,7 @@ default 60000 (1 minute)]</li> <li>$ActionQueueWorkerThreadMinumumMessages <number>, default 100</li> <li><a href="rsconf1_actionresumeinterval.html">$ActionResumeInterval</a></li> <li>$ActionResumeRetryCount <number> [default 0, -1 means eternal]</li> -<li>$ActionSendResendLastMsgOnReconn <[on/<b>off</b>]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. +<li>$ActionSendResendLastMsgOnReconnect <[on/<b>off</b>]> specifies if the last message is to be resend when a connecition breaks and has been reconnected. May increase reliability, but comes at the risk of message duplication. <li>$ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action</li> <li>$ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver (driver-specific)</li> <li>$ActionSendStreamDriverAuthMode <mode>, authentication mode to use with the stream driver. Note that this directive requires TLS diff --git a/doc/rsyslog_tls.html b/doc/rsyslog_tls.html index bb312c77..286660d2 100644 --- a/doc/rsyslog_tls.html +++ b/doc/rsyslog_tls.html @@ -162,25 +162,11 @@ similar "smart" command on the client. It should show up in the respective server log file. If you dig out your sniffer, you should see that the traffic on the wire is actually protected.</p> <h3>Limitations</h3> -<p>The current implementation has a number of limitations. These -are -being worked on. Most importantly, neither the client nor the server -are authenticated. So while the message transfer is encrypted, you can -not be sure which peer you are talking to. Please note that this is a -limitation found in most real-world SSL syslog systems. Of course, that -is not an excuse for not yet providing this feature - but it tells you -that it is acceptable and can be worked around by proper firewalling, -ACLs and other organizational measures. Mutual authentication will be -added shortly to rsyslog.</p> -<p>Secondly, the plain tcp syslog listener -can currently listen to a single port, in a single mode. So if you use -a TLS-based listener, you can not run unencrypted syslog on the same -instance at the same time. A work-around is to run a second rsyslogd -instance. This limitation, too, is scheduled to be removed soon.</p> <p>The RELP transport can currently not be protected by TLS. A work-around is to use stunnel. TLS support for RELP will be added once plain TCP -syslog has sufficiently matured.</p> +syslog has sufficiently matured and there either is some time left to do this +or we find a sponsor ;).</p> <h2>Certificates</h2> <p>In order to be really secure, certificates are needed. This is a short summary on how to generate the necessary certificates with diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 404cebc7..ed6ef509 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -205,7 +205,7 @@ doInjectMsg(int iNum) DEFiRet; snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), - "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum); + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum); datetime.getCurrTime(&stTime, &ttGenTime); /* we now create our own message object and submit it to the queue */ @@ -247,6 +247,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); + DBGPRINTF("imdiag: %d messages injected\n", nMsgs); finalize_it: RETiRet; @@ -279,6 +280,7 @@ waitMainQEmpty(tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "mainqueue empty\n")); + DBGPRINTF("imdiag: mainqueue empty\n"); finalize_it: RETiRet; @@ -314,6 +316,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); + DBGPRINTF("imdiag: %d messages in main queue\n", iMsgQueueSize); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) { CHKiRet(waitMainQEmpty(pSess)); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) { diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index aa98ae9d..cfbbad76 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -212,7 +212,9 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(statsobj, CORE_COMPONENT)); + /* the pstatsinverval is an alias to support a previous screwed-up syntax... */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatsinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatfacility", 0, eCmdHdlrInt, NULL, &cs.iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatseverity", 0, eCmdHdlrInt, NULL, &cs.iSeverity, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index d3e9cabe..1a62d82e 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -3,7 +3,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2011 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -88,6 +88,7 @@ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mos static int bEmitMsgOnClose = 0; /* emit an informational message on close by remote peer */ static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ static int bDisableLFDelim = 0; /* disbale standard LF delimiter */ +static int bUseFlowControl = 1; /* use flow control, what means indicate ourselfs a "light delayable" */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ @@ -199,6 +200,7 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); + CHKiRet(tcpsrv.SetUseFlowControl(pOurTcpsrv, bUseFlowControl)); CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); CHKiRet(tcpsrv.SetbDisableLFDelim(pOurTcpsrv, bDisableLFDelim)); CHKiRet(tcpsrv.SetNotificationOnRemoteClose(pOurTcpsrv, bEmitMsgOnClose)); @@ -289,6 +291,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus iTCPSessMax = 200; iTCPLstnMax = 20; iStrmDrvrMode = 0; + bUseFlowControl = 0; bEmitMsgOnClose = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; bDisableLFDelim = 0; @@ -344,6 +347,8 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverbindruleset"), 0, eCmdHdlrGetWord, setRuleset, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpflowcontrol"), 0, + eCmdHdlrBinary, NULL, &bUseFlowControl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index ab8e4d2c..ea4b4b75 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -149,8 +149,13 @@ static rsRetVal initPgSQL(instanceData *pData, int bSilent) dbgprintf("host=%s dbname=%s uid=%s\n",pData->f_dbsrv,pData->f_dbname,pData->f_dbuid); + /* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may + * get all sorts of side effects (e.g.: backslash escapes) and warnings + */ + const char *PgConnectionOptions = "-c standard_conforming_strings=on"; + /* Connect to database */ - if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, NULL, NULL, + if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, PgConnectionOptions, NULL, pData->f_dbname, pData->f_dbuid, pData->f_dbpwd)) == NULL) { reportDBError(pData, bSilent); closePgSQL(pData); /* ignore any error we may get */ diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index cf70381f..1453d68f 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -251,8 +251,18 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) /* extract the host first (we do a trick - we replace the ';' or ':' with a '\0') * now skip to port and then template name. rgerhards 2005-07-06 */ - for(q = p ; *p && *p != ';' && *p != ':' ; ++p) - /* JUST SKIP */; + if(*p == '[') { /* everything is hostname upto ']' */ + ++p; /* skip '[' */ + for(q = p ; *p && *p != ']' ; ++p) + /* JUST SKIP */; + if(*p == ']') { + *p = '\0'; /* trick to obtain hostname (later)! */ + ++p; /* eat it */ + } + } else { /* traditional view of hostname */ + for(q = p ; *p && *p != ';' && *p != ':' && *p != '#' ; ++p) + /* JUST SKIP */; + } pData->port = NULL; if(*p == ':') { /* process port */ diff --git a/plugins/sm_cust_bindcdr/Makefile.am b/plugins/sm_cust_bindcdr/Makefile.am new file mode 100644 index 00000000..1f71d499 --- /dev/null +++ b/plugins/sm_cust_bindcdr/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = sm_cust_bindcdr.la + +sm_cust_bindcdr_la_SOURCES = sm_cust_bindcdr.c +sm_cust_bindcdr_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +sm_cust_bindcdr_la_LDFLAGS = -module -avoid-version +sm_cust_bindcdr_la_LIBADD = diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c new file mode 100644 index 00000000..baad667e --- /dev/null +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -0,0 +1,390 @@ +/* sm_cust_bindcdr.c + * This is a custom developed plugin to process bind information into + * a specific SQL statement. While the actual processing may be too specific + * to be of general use, this module serves as a template on how this type + * of processing can be done. + * + * Format generated: + * "%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" + * Note that this is the same as smtradfile.c, except that we do have a RFC3339 timestamp. However, + * we have copied over the code from there, it is too simple to go through all the hassle + * of having a single code base. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2011-03-17 by RGerhards + * + * Copyright 2011 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <ctype.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "conf.h" +#include "syslogd-types.h" +#include "cfsysline.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" +#include "errmsg.h" + +MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP +STRGEN_NAME("Custom_BindCDR,sql") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +/* list of "allowed" IPs */ +typedef struct allowedip_s { + uchar *pszIP; + struct allowedip_s *next; +} allowedip_t; + +static allowedip_t *root; + + +/* config data */ + +/* check if the provided IP is (already) in the allowed list + */ +static int +isAllowed(uchar *pszIP) +{ + allowedip_t *pallow; + int ret = 0; + + for(pallow = root ; pallow != NULL ; pallow = pallow->next) { + if(!ustrcmp(pallow->pszIP, pszIP)) { + ret = 1; + goto finalize_it; + } + } +finalize_it: return ret; +} + +/* This function is called to add an additional allowed IP. It adds + * the IP to the linked list of them. An error is emitted if the IP + * already exists. + */ +static rsRetVal addAllowedIP(void __attribute__((unused)) *pVal, uchar *pNewVal) +{ + allowedip_t *pNew; + DEFiRet; + + if(isAllowed(pNewVal)) { + errmsg.LogError(0, NO_ERRCODE, "error: allowed IP '%s' already configured " + "duplicate ignored", pNewVal); + ABORT_FINALIZE(RS_RET_ERR); + } + + CHKmalloc(pNew = malloc(sizeof(allowedip_t))); + pNew->pszIP = pNewVal; + pNew->next = root; + root = pNew; + DBGPRINTF("sm_cust_bindcdr: allowed IP '%s' added.\n", pNewVal); + +finalize_it: + if(iRet != RS_RET_OK) { + free(pNewVal); + } + + RETiRet; +} + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + * An actual message sample for what we intend to parse is (one line): + <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96) + */ +//previos dev: #define SQL_STMT "INSERT INTO CDR(`Date`,`Time`, timeMS, client, view, query, ip) VALUES ('" +#define SQL_STMT "INSERT INTO CDR(`date`,ip,user,dest) VALUES ('" +#define ADD_SQL_DELIM \ + memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \ + iBuf += sizeof("', '") - 1; +#define SQL_STMT_END "');\n" +BEGINstrgen + int iBuf; + uchar *psz; + uchar szDate[64]; + unsigned lenDate; + uchar szTime[64]; + unsigned lenTime; + uchar szMSec[64]; + unsigned lenMSec; + uchar szClient[64]; + unsigned lenClient; + uchar szView[64]; + unsigned lenView; + uchar szQuery[64]; + unsigned lenQuery; + uchar szIP[64]; + unsigned lenIP; + size_t lenTotal; +CODESTARTstrgen + /* first create an empty statement. This is to be replaced if + * we have better data to fill in. + */ + /* now make sure buffer is large enough */ + if(*pLenBuf < 2) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, 2)); + memcpy(*ppBuf, ";", sizeof(";")); + + /* first obtain all strings and their length (if not fixed) */ + /* Note that there are two date fields present, one in the header + * and one more in the actual message. We use the one from the message + * and parse that our. We check validity based on some fixe fields. In- + * depth verification is probably not worth the effort (CPU time), because + * we do various other checks on the message format below). + */ + psz = getMSG(pMsg); + if(psz[0] == ' ' && psz[3] == '-' && psz[7] == '-') { + memcpy(szDate, psz+8, 4); + szDate[4] = '-'; + if(!strncmp((char*)psz+4, "Jan", 3)) { + szDate[5] = '0'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Feb", 3)) { + szDate[5] = '0'; + szDate[6] = '2'; + } else if(!strncmp((char*)psz+4, "Mar", 3)) { + szDate[5] = '0'; + szDate[6] = '3'; + } else if(!strncmp((char*)psz+4, "Apr", 3)) { + szDate[5] = '0'; + szDate[6] = '4'; + } else if(!strncmp((char*)psz+4, "May", 3)) { + szDate[5] = '0'; + szDate[6] = '5'; + } else if(!strncmp((char*)psz+4, "Jun", 3)) { + szDate[5] = '0'; + szDate[6] = '6'; + } else if(!strncmp((char*)psz+4, "Jul", 3)) { + szDate[5] = '0'; + szDate[6] = '7'; + } else if(!strncmp((char*)psz+4, "Aug", 3)) { + szDate[5] = '0'; + szDate[6] = '8'; + } else if(!strncmp((char*)psz+4, "Sep", 3)) { + szDate[5] = '0'; + szDate[6] = '9'; + } else if(!strncmp((char*)psz+4, "Oct", 3)) { + szDate[5] = '1'; + szDate[6] = '0'; + } else if(!strncmp((char*)psz+4, "Nov", 3)) { + szDate[5] = '1'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Dec", 3)) { + szDate[5] = '1'; + szDate[6] = '2'; + } + szDate[7] = '-'; + szDate[8] = psz[1]; + szDate[9] = psz[2]; + szDate[10] = '\0'; + lenDate = 10; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + + /* now time (pull both regular time and ms) */ + if(psz[12] == ' ' && psz[15] == ':' && psz[18] == ':' && psz[21] == '.' && psz[25] == ' ') { + memcpy(szTime, (char*)psz+13, 8); + szTime[9] = '\0'; + lenTime = 8; + memcpy(szMSec, (char*)psz+22, 3); + szMSec[4] = '\0'; + lenMSec = 3; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + + /* "client" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: client part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("client ") - 1; /* skip "label" */ + for( lenClient = 0 + ; *psz && *psz != '#' && lenClient < sizeof(szClient) - 1 + ; ++lenClient) { + szClient[lenClient] = *psz++; + } + szClient[lenClient] = '\0'; + } + + /* "view" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "view "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: view part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("view ") - 1; /* skip "label" */ + for( lenView = 0 + ; *psz && *psz != ':' && lenView < sizeof(szView) - 1 + ; ++lenView) { + szView[lenView] = *psz++; + } + szView[lenView] = '\0'; + } + + /* "query" - we must extract just the number, and in reverse! */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "query: "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: query part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("query: ") - 1; /* skip "label" */ + /* first find end-of-strihttp://www.rsyslog.com/doc/omruleset.htmlng to process */ + while(*psz && (isdigit(*psz) || *psz == '.')) { + psz++; + } + /* now shuffle data */ + for( lenQuery = 0 + ; *psz && *psz != ' ' && lenQuery < sizeof(szQuery) - 1 + ; --psz) { + if(isdigit(*psz)) + szQuery[lenQuery++] = *psz; + } + szQuery[lenQuery] = '\0'; + } + + /* "ip" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + ("); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: ip part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("IN TXT + (") - 1; /* skip "label" */ + for( lenIP = 0 + ; *psz && *psz != ')' && lenIP < sizeof(szIP) - 1 + ; ++lenIP) { + szIP[lenIP] = *psz++; + } + szIP[lenIP] = '\0'; + } + + + /* --- strings extracted ---- */ + + /* now check if the IP is "allowed", in which case we should not + * insert into the database. + */ + if(isAllowed(szIP)) { + DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = lenDate + lenTime + lenMSec + lenClient + lenView + lenQuery + + lenIP + 7 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1); + iBuf = sizeof(SQL_STMT) - 1; + + memcpy(*ppBuf + iBuf, szDate, lenDate); + iBuf += lenDate; + /* prviously: ADD_SQL_DELIM */ + *(*ppBuf + iBuf) = ' '; + ++iBuf; + + memcpy(*ppBuf + iBuf, szTime, lenTime); + iBuf += lenTime; + ADD_SQL_DELIM + + /* we shall now discard this part + memcpy(*ppBuf + iBuf, szMSec, lenMSec); + iBuf += lenMSec; + ADD_SQL_DELIM + */ + + /* Note that this seem to be the IP to use */ + memcpy(*ppBuf + iBuf, szClient, lenClient); + iBuf += lenClient; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szView, lenView); + iBuf += lenView; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szQuery, lenQuery); + iBuf += lenQuery; + /* this is now the last field, so we dont need: ADD_SQL_DELIM */ + + /* no longer to be included + memcpy(*ppBuf + iBuf, szIP, lenIP); + iBuf += lenIP; + */ + + /* end of SQL statement/trailer (NUL is contained in string!) */ + memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END)); + iBuf += sizeof(SQL_STMT_END); + +finalize_it: +ENDstrgen + + +BEGINmodExit + allowedip_t *pallow, *pdel; +CODESTARTmodExit + for(pallow = root ; pallow != NULL ; ) { + pdel = pallow; + pallow = pallow->next; + free(pdel->pszIP); + free(pdel); + } + + objRelease(errmsg, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + root = NULL; + CHKiRet(omsdRegCFSLineHdlr((uchar *)"sgcustombindcdrallowedip", 0, eCmdHdlrGetWord, + addAllowedIP, NULL, STD_LOADABLE_MODULE_ID)); + dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION); +ENDmodInit diff --git a/runtime/msg.c b/runtime/msg.c index b0261faa..d1e67aa2 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -287,6 +287,121 @@ static pthread_mutex_t mutTrimCtr; /* mutex to handle malloc trim */ static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); +/* The following functions will support advanced output module + * multithreading, once this is implemented. Currently, we + * include them as hooks only. The idea is that we need to guard + * some msg objects data fields against concurrent access if + * we run on multiple threads. Please note that in any case this + * is not necessary for calls from INPUT modules, because they + * construct the message object and do this serially. Only when + * the message is in the processing queue, multiple threads may + * access a single object. Consequently, there are no guard functions + * for "set" methods, as these are called during input. Only "get" + * functions that modify important structures have them. + * rgerhards, 2007-07-20 + * We now support locked and non-locked operations, depending on + * the configuration of rsyslog. To support this, we use function + * pointers. Initially, we start in non-locked mode. There, all + * locking operations call into dummy functions. When locking is + * enabled, the function pointers are changed to functions doing + * actual work. We also introduced another MsgPrepareEnqueue() function + * which initializes the locking structures, if needed. This is + * necessary because internal messages during config file startup + * processing are always created in non-locking mode. So we can + * not initialize locking structures during constructions. We now + * postpone this until when the message is fully constructed and + * enqueued. Then we know the status of locking. This has a nice + * side effect, and that is that during the initial creation of + * the Msg object no locking needs to be done, which results in better + * performance. -- rgerhards, 2008-01-05 + */ +static void (*funcLock)(msg_t *pMsg); +static void (*funcUnlock)(msg_t *pMsg); +static void (*funcDeleteMutex)(msg_t *pMsg); +void (*funcMsgPrepareEnqueue)(msg_t *pMsg); +#if 1 /* This is a debug aid */ +#define MsgLock(pMsg) funcLock(pMsg) +#define MsgUnlock(pMsg) funcUnlock(pMsg) +#else +#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; } +#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); } +#endif + +/* the next function is a dummy to be used by the looking functions + * when the class is not yet running in an environment where locking + * is necessary. Please note that the need to lock can (and will) change + * during a single run. Typically, this is depending on the operation mode + * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05 + */ +static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) +{ + /* empty be design */ +} + + +/* The following function prepares a message for enqueue into the queue. This is + * where a message may be accessed by multiple threads. This implementation here + * is the version for multiple concurrent acces. It initializes the locking + * structures. + * TODO: change to an iRet interface! -- rgerhards, 2008-07-14 + */ +static void MsgPrepareEnqueueLockingCase(msg_t *pThis) +{ + BEGINfunc + assert(pThis != NULL); + pthread_mutex_init(&pThis->mut, NULL); + pThis->bDoLock = 1; + ENDfunc +} + + +/* ... and now the locking and unlocking implementations: */ +static void MsgLockLockingCase(msg_t *pThis) +{ + /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ + assert(pThis != NULL); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_lock(&pThis->mut); +} + +static void MsgUnlockLockingCase(msg_t *pThis) +{ + /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ + assert(pThis != NULL); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_unlock(&pThis->mut); +} + +/* delete the mutex object on message destruction (locking case) + */ +static void MsgDeleteMutexLockingCase(msg_t *pThis) +{ + assert(pThis != NULL); + pthread_mutex_destroy(&pThis->mut); +} + +/* enable multiple concurrent access on the message object + * This works on a class-wide basis and can bot be undone. + * That is, if it is once enabled, it can not be disabled during + * the same run. When this function is called, no other thread + * must manipulate message objects. Then we would have race conditions, + * but guarding against this is counter-productive because it + * would cost additional time. Plus, it would be a programming error. + * rgerhards, 2008-01-05 + */ +rsRetVal MsgEnableThreadSafety(void) +{ + DEFiRet; + funcLock = MsgLockLockingCase; + funcUnlock = MsgUnlockLockingCase; + funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase; + funcDeleteMutex = MsgDeleteMutexLockingCase; + RETiRet; +} + +/* end locking functions */ + + static inline int getProtocolVersion(msg_t *pM) { return(pM->iProtocolVersion); @@ -306,6 +421,7 @@ resolveDNS(msg_t *pMsg) { uchar fromHostFQDN[NI_MAXHOST]; DEFiRet; + MsgLock(pMsg); CHKiRet(objUse(net, CORE_COMPONENT)); if(pMsg->msgFlags & NEEDS_DNSRESOL) { localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); @@ -315,6 +431,7 @@ resolveDNS(msg_t *pMsg) { } } finalize_it: + MsgUnlock(pMsg); if(iRet != RS_RET_OK) { /* best we can do: remove property */ MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); @@ -444,6 +561,8 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) *pPropID = PROP_SYS_MINUTE; } else if(!strcmp((char*) pName, "$myhostname")) { *pPropID = PROP_SYS_MYHOSTNAME; + } else if(!strcmp((char*) pName, "$bom")) { + *pPropID = PROP_SYS_BOM; } else { *pPropID = PROP_INVALID; iRet = RS_RET_VAR_NOT_FOUND; @@ -525,127 +644,14 @@ uchar *propIDToName(propid_t propID) return UCHAR_CONSTANT("$MINUTE"); case PROP_SYS_MYHOSTNAME: return UCHAR_CONSTANT("$MYHOSTNAME"); + case PROP_SYS_BOM: + return UCHAR_CONSTANT("$BOM"); default: return UCHAR_CONSTANT("*invalid property id*"); } } -/* The following functions will support advanced output module - * multithreading, once this is implemented. Currently, we - * include them as hooks only. The idea is that we need to guard - * some msg objects data fields against concurrent access if - * we run on multiple threads. Please note that in any case this - * is not necessary for calls from INPUT modules, because they - * construct the message object and do this serially. Only when - * the message is in the processing queue, multiple threads may - * access a single object. Consequently, there are no guard functions - * for "set" methods, as these are called during input. Only "get" - * functions that modify important structures have them. - * rgerhards, 2007-07-20 - * We now support locked and non-locked operations, depending on - * the configuration of rsyslog. To support this, we use function - * pointers. Initially, we start in non-locked mode. There, all - * locking operations call into dummy functions. When locking is - * enabled, the function pointers are changed to functions doing - * actual work. We also introduced another MsgPrepareEnqueue() function - * which initializes the locking structures, if needed. This is - * necessary because internal messages during config file startup - * processing are always created in non-locking mode. So we can - * not initialize locking structures during constructions. We now - * postpone this until when the message is fully constructed and - * enqueued. Then we know the status of locking. This has a nice - * side effect, and that is that during the initial creation of - * the Msg object no locking needs to be done, which results in better - * performance. -- rgerhards, 2008-01-05 - */ -static void (*funcLock)(msg_t *pMsg); -static void (*funcUnlock)(msg_t *pMsg); -static void (*funcDeleteMutex)(msg_t *pMsg); -void (*funcMsgPrepareEnqueue)(msg_t *pMsg); -#if 1 /* This is a debug aid */ -#define MsgLock(pMsg) funcLock(pMsg) -#define MsgUnlock(pMsg) funcUnlock(pMsg) -#else -#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; } -#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); } -#endif - -/* the next function is a dummy to be used by the looking functions - * when the class is not yet running in an environment where locking - * is necessary. Please note that the need to lock can (and will) change - * during a single run. Typically, this is depending on the operation mode - * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05 - */ -static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) -{ - /* empty be design */ -} - - -/* The following function prepares a message for enqueue into the queue. This is - * where a message may be accessed by multiple threads. This implementation here - * is the version for multiple concurrent acces. It initializes the locking - * structures. - * TODO: change to an iRet interface! -- rgerhards, 2008-07-14 - */ -static void MsgPrepareEnqueueLockingCase(msg_t *pThis) -{ - BEGINfunc - assert(pThis != NULL); - pthread_mutex_init(&pThis->mut, NULL); - pThis->bDoLock = 1; - ENDfunc -} - - -/* ... and now the locking and unlocking implementations: */ -static void MsgLockLockingCase(msg_t *pThis) -{ - /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ - assert(pThis != NULL); - if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ - pthread_mutex_lock(&pThis->mut); -} - -static void MsgUnlockLockingCase(msg_t *pThis) -{ - /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ - assert(pThis != NULL); - if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ - pthread_mutex_unlock(&pThis->mut); -} - -/* delete the mutex object on message destruction (locking case) - */ -static void MsgDeleteMutexLockingCase(msg_t *pThis) -{ - assert(pThis != NULL); - pthread_mutex_destroy(&pThis->mut); -} - -/* enable multiple concurrent access on the message object - * This works on a class-wide basis and can bot be undone. - * That is, if it is once enabled, it can not be disabled during - * the same run. When this function is called, no other thread - * must manipulate message objects. Then we would have race conditions, - * but guarding against this is counter-productive because it - * would cost additional time. Plus, it would be a programming error. - * rgerhards, 2008-01-05 - */ -rsRetVal MsgEnableThreadSafety(void) -{ - DEFiRet; - funcLock = MsgLockLockingCase; - funcUnlock = MsgUnlockLockingCase; - funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase; - funcDeleteMutex = MsgDeleteMutexLockingCase; - RETiRet; -} - -/* end locking functions */ - - /* This is common code for all Constructors. It is defined in an * inline'able function so that we can save a function call in the * actual constructors (otherwise, the msgConstruct would need @@ -2427,6 +2433,12 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, case PROP_SYS_MYHOSTNAME: pRes = glbl.GetLocalHostName(); break; + case PROP_SYS_BOM: + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (uchar*) "\xEF\xBB\xBF"; + *pbMustBeFreed = 0; + break; default: /* there is no point in continuing, we may even otherwise render the * error message unreadable. rgerhards, 2007-07-10 diff --git a/runtime/parser.c b/runtime/parser.c index d3644976..b385c54b 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -453,10 +453,10 @@ ParsePRI(msg_t *pMsg) if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) pri = DEFUPRI; } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); } - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); - MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); RETiRet; } diff --git a/runtime/queue.c b/runtime/queue.c index ef6e843b..50ae307c 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1774,9 +1774,13 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) { DEFiRet; +//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis)); if(pThis->bEnqOnly) { iRet = RS_RET_TERMINATE_WHEN_IDLE; } + if(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk) { + iRet = RS_RET_TERMINATE_NOW; + } RETiRet; } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 78e61bcb..d63dbe4f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -137,6 +137,7 @@ typedef uintTiny propid_t; #define PROP_SYS_QHOUR 156 #define PROP_SYS_MINUTE 157 #define PROP_SYS_MYHOSTNAME 158 +#define PROP_SYS_BOM 159 /* The error codes below are orginally "borrowed" from diff --git a/runtime/rule.c b/runtime/rule.c index 42773768..d5f18e71 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -173,7 +173,7 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, sbool *bProcessMsg) if(pRule->f_filter_type == FILTER_PRI) { /* skip messages that are incorrect priority */ -dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility]); + dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility]); if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ ((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) ) bRet = 0; @@ -185,7 +185,7 @@ dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFac CHKiRet(vm.SetMsg(pVM, pMsg)); CHKiRet(vm.ExecProg(pVM, pRule->f_filterData.f_expr->pVmprg)); CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + dbgprintf("result of rainerscript filter evaluation: %lld\n", pResult->val.num); /* VM is destructed on function exit */ bRet = (pResult->val.num) ? 1 : 0; } else { diff --git a/runtime/ruleset.c b/runtime/ruleset.c index c9c64a38..5ee2a55a 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -223,7 +223,7 @@ processBatch(batch_t *pBatch) DEFiRet; assert(pBatch != NULL); -dbgprintf("ZZZ: processBatch: batch of %d elements must be processed\n", pBatch->nElem); + DBGPRINTF("processBatch: batch of %d elements must be processed\n", pBatch->nElem); if(pBatch->bSingleRuleset) { pThis = batchGetRuleset(pBatch); if(pThis == NULL) @@ -235,7 +235,7 @@ dbgprintf("ZZZ: processBatch: batch of %d elements must be processed\n", pBatch- } finalize_it: -dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet); + DBGPRINTF("ruleset.ProcessMsg() returns %d\n", iRet); RETiRet; } diff --git a/runtime/sysvar.c b/runtime/sysvar.c index 4a6ace19..ecc31e2d 100644 --- a/runtime/sysvar.c +++ b/runtime/sysvar.c @@ -41,6 +41,7 @@ DEFobjStaticHelpers DEFobjCurrIf(var) DEFobjCurrIf(datetime) +DEFobjCurrIf(glbl) /* Standard-Constructor @@ -146,6 +147,8 @@ GetVar(cstr_t *pstrVarName, var_t **ppVar) CHKiRet(getNOW(NOW_HOUR, &pstrProp)); } else if(!rsCStrSzStrCmp(pstrVarName, (uchar*)"minute", sizeof("minute") - 1)) { CHKiRet(getNOW(NOW_MINUTE, &pstrProp)); + } else if(!rsCStrSzStrCmp(pstrVarName, (uchar*)"myhostname", sizeof("myhostname") - 1)) { + CHKiRet(rsCStrConstructFromszStr(&pstrProp, glbl.GetLocalHostName())); } else { ABORT_FINALIZE(RS_RET_SYSVAR_NOT_FOUND); } @@ -191,6 +194,7 @@ BEGINObjClassInit(sysvar, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, sysvarConstructFinalize); diff --git a/runtime/vm.c b/runtime/vm.c index 0ed174d1..84ba4bcf 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -288,7 +288,7 @@ BEGINCMPOP(CMP_NEQ) /* remember to change the name also in the END macro! */ case VARTYPE_STR: bRes = rsCStrCStrCmp(operand1->val.pStr, operand2->val.pStr); break; -ENDCMPOP(CMP_EQ) +ENDCMPOP(CMP_NEQ) BEGINCMPOP(CMP_LT) /* remember to change the name also in the END macro! */ case VARTYPE_NUMBER: @@ -474,6 +474,15 @@ CODESTARTop(PUSHSYSVAR) CHKiRet(sysvar.GetVar(pOp->operand.pVar->val.pStr, &pVal)); vmstk.Push(pThis->pStk, pVal); finalize_it: + if(Debug && iRet != RS_RET_OK) { + if(iRet == RS_RET_SYSVAR_NOT_FOUND) { + DBGPRINTF("rainerscript: sysvar '%s' not found\n", + rsCStrGetSzStrNoNULL(pOp->operand.pVar->val.pStr)); + } else { + DBGPRINTF("rainerscript: error %d trying to obtain sysvar '%s'\n", + iRet, rsCStrGetSzStrNoNULL(pOp->operand.pVar->val.pStr)); + } + } ENDop(PUSHSYSVAR) /* The function call operation is only very roughly implemented. While the plumbing @@ -666,9 +675,11 @@ execProg(vm_t *pThis, vmprg_t *pProg) ISOBJ_TYPE_assert(pThis, vm); ISOBJ_TYPE_assert(pProg, vmprg); -#define doOP(OP) case opcode_##OP: CHKiRet(op##OP(pThis, pCurrOp)); break +#define doOP(OP) case opcode_##OP: DBGPRINTF("rainerscript: opcode %s\n", #OP); \ + CHKiRet(op##OP(pThis, pCurrOp)); break pCurrOp = pProg->vmopRoot; /* TODO: do this via a method! */ while(pCurrOp != NULL && pCurrOp->opcode != opcode_END_PROG) { + DBGPRINTF("rainerscript: executing step, opcode %d...\n", pCurrOp->opcode); switch(pCurrOp->opcode) { doOP(OR); doOP(AND); @@ -695,8 +706,8 @@ execProg(vm_t *pThis, vmprg_t *pProg) doOP(UNARY_MINUS); doOP(FUNC_CALL); default: - ABORT_FINALIZE(RS_RET_INVALID_VMOP); dbgoprint((obj_t*) pThis, "invalid instruction %d in vmprg\n", pCurrOp->opcode); + ABORT_FINALIZE(RS_RET_INVALID_VMOP); break; } /* so far, we have plain sequential execution, so on to next... */ @@ -709,6 +720,7 @@ execProg(vm_t *pThis, vmprg_t *pProg) */ finalize_it: + DBGPRINTF("rainerscript: script execution terminated with state %d\n", iRet); RETiRet; } diff --git a/tcps_sess.c b/tcps_sess.c index 99af0cb8..bed598dd 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -253,7 +253,8 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); MsgSetInputName(pMsg, pThis->pLstnInfo->pInputName); - MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); + MsgSetFlowControlType(pMsg, pThis->pSrv->bUseFlowControl + ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; MsgSetRcvFrom(pMsg, pThis->fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); @@ -716,6 +716,7 @@ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macr pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; pThis->bDisableLFDelim = 0; pThis->OnMsgReceive = NULL; + pThis->bUseFlowControl = 1; ENDobjConstruct(tcpsrv) @@ -991,6 +992,18 @@ SetLstnMax(tcpsrv_t *pThis, int iMax) } +/* set if flow control shall be supported + */ +static rsRetVal +SetUseFlowControl(tcpsrv_t *pThis, int bUseFlowControl) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->bUseFlowControl = bUseFlowControl; + RETiRet; +} + + /* set max number of sessions * this must be called before ConstructFinalize, or it will have no effect! * rgerhards, 2009-04-09 @@ -1033,6 +1046,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; pIf->SetbDisableLFDelim = SetbDisableLFDelim; pIf->SetSessMax = SetSessMax; + pIf->SetUseFlowControl = SetUseFlowControl; pIf->SetLstnMax = SetLstnMax; pIf->SetDrvrMode = SetDrvrMode; pIf->SetDrvrAuthMode = SetDrvrAuthMode; @@ -56,6 +56,7 @@ struct tcpsrv_s { permittedPeers_t *pPermPeers;/**< driver's permitted peers */ sbool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ sbool bUsingEPoll; /**< are we in epoll mode (means we do not need to keep track of sessions!) */ + sbool bUseFlowControl; /**< use flow control (make light delayable) */ int iLstnCurr; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ @@ -121,8 +122,10 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetNotificationOnRemoteClose)(tcpsrv_t *pThis, int bNewVal); /* 2009-10-01 */ /* added v9 -- rgerhards, 2010-03-01 */ rsRetVal (*SetbDisableLFDelim)(tcpsrv_t*, int); + /* added v10 -- rgerhards, 2011-04-01 */ + rsRetVal (*SetUseFlowControl)(tcpsrv_t*, int); ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 10 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 @@ -42,10 +42,13 @@ /* static data */ DEFobjCurrIf(obj) DEFobjCurrIf(errmsg) -DEFobjCurrIf(regexp) DEFobjCurrIf(strgen) +#ifdef FEATURE_REGEXP +DEFobjCurrIf(regexp) static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */ +#endif + static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ @@ -549,10 +552,9 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) cstr_t *pStrB; struct templateEntry *pTpe; int iNum; /* to compute numbers */ - rsRetVal iRetLocal; - #ifdef FEATURE_REGEXP /* APR: variables for regex */ + rsRetVal iRetLocal; int longitud; unsigned char *regex_char; unsigned char *regex_end; @@ -856,19 +858,37 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) { uchar *pSrc, *pDst; uchar szMod[2048]; + unsigned lenMod; strgen_t *pStrgen; DEFiRet; pSrc = *ppRestOfConfLine; pDst = szMod; - while(*pSrc && !isspace(*pSrc) && pDst < &(szMod[sizeof(szMod) - 1])) { - *pDst++ = *pSrc++; + lenMod = 0; + while(*pSrc && !isspace(*pSrc) && lenMod < sizeof(szMod) - 1) { + szMod[lenMod] = *pSrc++; + lenMod++; + } - *pDst = '\0'; + szMod[lenMod] = '\0'; *ppRestOfConfLine = pSrc; CHKiRet(strgen.FindStrgen(&pStrgen, szMod)); pTpl->pStrgen = pStrgen->pModule->mod.sm.strgen; - dbgprintf("template bound to strgen '%s'\n", szMod); + DBGPRINTF("template bound to strgen '%s'\n", szMod); + /* check if the name potentially contains some well-known options + * Note: we have opted to let the name contain all options. This sounds + * useful, because the strgen MUST actually implement a specific set + * of options. Doing this via the name looks to the enduser as if the + * regular syntax were used, and it make sure the strgen postively + * acknowledged implementing the option. -- rgerhards, 2011-03-21 + */ + if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) { + pTpl->optFormatForSQL = 2; + DBGPRINTF("strgen suports the stdsql option\n"); + } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) { + pTpl->optFormatForSQL = 1; + DBGPRINTF("strgen suports the sql option\n"); + } finalize_it: RETiRet; @@ -936,8 +956,7 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) /* we simply make the template defunct in this case by setting * its name to a zero-string. We do not free it, as this would * require additional code and causes only a very small memory - * consumption. Memory is freed, however, in normal operation - * and most importantly by HUPing syslogd. + * consumption. */ *pTpl->pszName = '\0'; return NULL; @@ -1067,11 +1086,13 @@ void tplDeleteAll(void) break; case FIELD: /* check if we have a regexp and, if so, delete it */ +#ifdef FEATURE_REGEXP if(pTpeDel->data.field.has_regex != 0) { if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { regexp.regfree(&(pTpeDel->data.field.re)); } } +#endif break; } /*dbgprintf("\n");*/ @@ -1120,12 +1141,14 @@ void tplDeleteNew(void) free(pTpeDel->data.constant.pConstant); break; case FIELD: +#ifdef FEATURE_REGEXP /* check if we have a regexp and, if so, delete it */ if(pTpeDel->data.field.has_regex != 0) { if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { regexp.regfree(&(pTpeDel->data.field.re)); } } +#endif break; } /*dbgprintf("\n");*/ diff --git a/tests/Makefile.am b/tests/Makefile.am index ebbe0cf6..930aa304 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -56,8 +56,7 @@ TESTS += \ failover-rptd.sh \ failover-no-rptd.sh \ failover-no-basic.sh \ - queue-persist.sh - queue-persist.sh \ + rcvr_fail_restore.sh \ linkedlistqueue.sh endif @@ -289,6 +288,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ diag.sh \ testsuites/diag-common.conf \ testsuites/diag-common2.conf \ + rcvr_fail_restore.sh \ + testsuites/rcvr_fail_restore_rcvr.conf \ + testsuites/rcvr_fail_restore_sender.conf \ daqueue-persist.sh \ daqueue-persist-drvr.sh \ queue-persist.sh \ diff --git a/tests/diag.sh b/tests/diag.sh index 8fbe42ed..d1242fb1 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -84,15 +84,27 @@ case $1 in exit 1 fi ;; + 'get-mainqueuesize') # show the current main queue size + if [ "$2" == "2" ] + then + echo getmainmsgqueuesize | ./diagtalker -p13501 + else + echo getmainmsgqueuesize | ./diagtalker + fi + ;; 'wait-queueempty') # wait for main message queue to be empty. $2 is the instance. if [ "$2" == "2" ] then - echo WaitMainQueueEmpty | ./diagtalker + echo WaitMainQueueEmpty | ./diagtalker -p13501 else echo WaitMainQueueEmpty | ./diagtalker fi ;; 'shutdown-when-empty') # shut rsyslogd down when main queue is empty. $2 is the instance. + if [ "$2" == "2" ] + then + echo Shutting down instance 2 + fi $srcdir/diag.sh wait-queueempty $2 kill `cat rsyslog$2.pid` # note: we do not wait for the actual termination! diff --git a/tests/diagtalker.c b/tests/diagtalker.c index 6a721e47..0d728bda 100644 --- a/tests/diagtalker.c +++ b/tests/diagtalker.c @@ -1,7 +1,7 @@ /* A yet very simple tool to talk to imdiag (this replaces the * previous Java implementation in order to get fewer dependencies). * - * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2010,2011 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -126,7 +126,7 @@ doProcessing() len = strlen(line); sendCmd(fd, line, len); waitRsp(fd, line, sizeof(line)); - printf("imdiag: %s", line); + printf("imdiag[%d]: %s", targetPort, line); } } @@ -139,7 +139,7 @@ int main(int argc, char *argv[]) int ret = 0; int opt; - while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) { + while((opt = getopt(argc, argv, "t:p:")) != -1) { switch (opt) { case 't': targetIP = optarg; break; diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh new file mode 100755 index 00000000..79486f10 --- /dev/null +++ b/tests/rcvr_fail_restore.sh @@ -0,0 +1,122 @@ +# Copyright (C) 2011 by Rainer Gerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[rcvr_fail_restore.sh\]: test failed receiver restore case +source $srcdir/diag.sh init +# +# STEP1: start both instances and send 1000 messages. +# Note: receiver is instance 2, sender instance 1. +# +# start up the instances. Note that the envrionment settings can be changed to +# set instance-specific debugging parameters! +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="log2" +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +#export RSYSLOG_DEBUGLOG="log" +#valgrind="valgrind" +source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf +# re-set params so that new instances do not thrash it... +#unset RSYSLOG_DEBUG +#unset RSYSLOG_DEBUGLOG + +# now inject the messages into instance 2. It will connect to instance 1, +# and that instance will record the data. +source $srcdir/diag.sh injectmsg 1 1000 +source $srcdir/diag.sh wait-queueempty +./msleep 1000 # let things settle down a bit + +# +# Step 2: shutdown receiver, then send some more data, which then +# needs to go into the queue. +# + +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +source $srcdir/diag.sh injectmsg 1001 10000 +./msleep 3000 # make sure some retries happen (retry interval is set to 3 second) +source $srcdir/diag.sh get-mainqueuesize +ls -l test-spool + +# +# Step 3: restart receiver, wait that the sender drains its queue +# +#export RSYSLOG_DEBUGLOG="log2" +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +echo waiting for sender to drain queue [may need a short while] +source $srcdir/diag.sh wait-queueempty +ls -l test-spool +OLDFILESIZE=$(stat -c%s test-spool/mainq.00000001) +echo file size to expect is $OLDFILESIZE + + +# +# Step 4: send new data. Queue files are not permitted to grow now +# (but one file continous to exist). +# +source $srcdir/diag.sh injectmsg 11001 10 +source $srcdir/diag.sh wait-queueempty + +# at this point, the queue file shall not have grown. Note +# that we MUST NOT shut down the instance right now, because it +# would clean up the queue files! So we need to do our checks +# first (here!). +ls -l test-spool +NEWFILESIZE=$(stat -c%s test-spool/mainq.00000001) +if [ $NEWFILESIZE != $OLDFILESIZE ] +then + echo file sizes do not match, expected $OLDFILESIZE, actual $NEWFILESIZE + echo this means that data has been written to the queue file where it + echo no longer should be written. + # abort will happen below, because we must ensure proper system shutdown + # HOWEVER, during actual testing it may be useful to do an exit here (so + # that e.g. the debug log is pointed right at the correct spot). + # exit 1 +fi + +# +# We now do an extra test (so this is two in one ;)) to see if the DA +# queue can be reactivated after its initial shutdown. In essence, we +# redo steps 2 and 3. +# +# Step 5: stop receiver again, then send some more data, which then +# needs to go into the queue. +# +echo "*** done primary test *** now checking if DA can be restarted" +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +source $srcdir/diag.sh injectmsg 11011 10000 +sleep 1 # we need to wait, otherwise we may be so fast that the receiver +# comes up before we have finally suspended the action +source $srcdir/diag.sh get-mainqueuesize +ls -l test-spool + +# +# Step 6: restart receiver, wait that the sender drains its queue +# +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +echo waiting for sender to drain queue [may need a short while] +source $srcdir/diag.sh wait-queueempty +ls -l test-spool + +# +# Queue file size checks done. Now it is time to terminate the system +# and see if everything could be received (the usual check, done here +# for completeness, more or less as a bonus). +# +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown + +# now it is time to stop the receiver as well +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +# now abort test if we need to (due to filesize predicate) +if [ $NEWFILESIZE != $OLDFILESIZE ] +then + exit 1 +fi +# do the final check +source $srcdir/diag.sh seq-check 1 21010 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf index 9e9e28fe..d76e2d15 100644 --- a/tests/testsuites/diag-common.conf +++ b/tests/testsuites/diag-common.conf @@ -13,4 +13,7 @@ $IMDiagServerRun 13500 $template startupfile,"rsyslogd.started" # trick to use relative path names! :syslogtag, contains, "rsyslogd" ?startupfile -$ErrorMessagesToStderr off +# I have disabled the directive below, so that we see errors in testcase +# creation. I am not sure why it was present in the first place, so for +# now I just leave it commented out -- rgerhards, 2011-03-30 +#$ErrorMessagesToStderr off diff --git a/tests/testsuites/random.conf b/tests/testsuites/random.conf index a7079df1..74ec61a8 100644 --- a/tests/testsuites/random.conf +++ b/tests/testsuites/random.conf @@ -4,6 +4,11 @@ # rgerhards, 2010-04-01 $IncludeConfig diag-common.conf +# The random data will generate TCP framing error messages. We will +# not clutter the test output with them. So we disable error messages +# to stderr. +$ErrorMessagesToStderr off + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 diff --git a/tests/testsuites/rcvr_fail_restore_rcvr.conf b/tests/testsuites/rcvr_fail_restore_rcvr.conf new file mode 100644 index 00000000..38ebad29 --- /dev/null +++ b/tests/testsuites/rcvr_fail_restore_rcvr.conf @@ -0,0 +1,8 @@ +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# then SENDER sends to this port (not tcpflood!) +$InputTCPServerRun 13515 + +$template outfmt,"%msg:F,58:2%\n" +:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/rcvr_fail_restore_sender.conf b/tests/testsuites/rcvr_fail_restore_sender.conf new file mode 100644 index 00000000..6b11aa4a --- /dev/null +++ b/tests/testsuites/rcvr_fail_restore_sender.conf @@ -0,0 +1,15 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +$WorkDirectory test-spool +$MainMsgQueueMaxFileSize 1g +$MainMsgQueueFileName mainq + +# we use the shortest resume interval a) to let the test not run too long +# and b) make sure some retries happen before the reconnect +$ActionResumeInterval 1 +$ActionResumeRetryCount -1 +*.* @@127.0.0.1:13515 diff --git a/tests/testsuites/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1 index 23836c9f..8a66d7ba 100644 --- a/tests/testsuites/rfc5424-1.parse1 +++ b/tests/testsuites/rfc5424-1.parse1 @@ -1,3 +1,3 @@ #Example from RFC5424, section 6.5 / sample 1 <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 -34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,BOM'su root' failed for lonvick on /dev/pts/8 diff --git a/tests/testsuites/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1 index a86fbc35..f5449e68 100644 --- a/tests/testsuites/rfc5424-2.parse1 +++ b/tests/testsuites/rfc5424-2.parse1 @@ -1,4 +1,4 @@ <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. -165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. +165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],%% It's time to make the do-nuts. #Example from RFC5424, section 6.5 / sample 2 #Only the first two lines are important, you may place anything behind them! diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c index 2bd18042..b06f1347 100644 --- a/tools/pmrfc5424.c +++ b/tools/pmrfc5424.c @@ -142,7 +142,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr * structured data. There may also be \] inside the structured data, which * do NOT terminate an element. */ - if(lenStr == 0 || *p2parse != '[') + if(lenStr == 0 || (*p2parse != '[' && *p2parse != '-')) return 1; /* this is NOT structured data! */ if(*p2parse == '-') { /* empty structured data? */ @@ -211,7 +211,6 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr * * rger, 2005-11-24 */ -//static int parseRFCSyslogMsg(msg_t *pMsg, int flags) BEGINparse uchar *p2parse; uchar *pBuf = NULL; |