diff options
66 files changed, 2726 insertions, 289 deletions
@@ -1,4 +1,69 @@ --------------------------------------------------------------------------- +Version 7.5.3 [devel] 2013-07-?? +- librelp 1.2.0 is now required +- make use of new librelp generic error reporting facility + This leads to more error messages being passed to the user and + thus simplified troubleshooting. +- bugfix: very small memory leak in imrelp + more or less cosmetic, a single memory block was not freed, but this + only happens immediately before termination (when the OS automatically + frees all memory). Still an annoyance e.g. in valgrind. +--------------------------------------------------------------------------- +Version 7.5.2 [devel] 2013-07-04 +- librelp 1.1.4 is now required + We use API extensions for better error reporting and higher performance. +- omrelp: use transactional mode to make imrelp emit bulk sends +- omrelp: add "windowSize" parameter to set custom RELP window size +- bugfix: double-free in omelasticsearch + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=461 + a security advisory for this bug is available at: + http://www.lsexperts.de/advisories/lse-2013-07-03.txt + CVE: CVE-2013-4758 + PLEASE NOTE: This issue only existed if omelasticsearch was used + in a non-default configuration, where the "errorfile" parameter + was specified. Without that parameter set, the bug could not + be triggered. + Thanks to Markus Vervier and Marius Ionescu for providing a detailled + bug report. Special thanks to Markus for coordinating his security + advisory with us. +- doc: fixed various typos + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=391 + Thanks to Georgi Georgiev for the patch. +--------------------------------------------------------------------------- +Version 7.5.1 [devel] 2013-06-26 +- librelp 1.1.3 is required - older versions can lead to a segfault +- add mmfields, which among others supports easy parsing of CEF messages +- omrelp: + * new parameter "compression.prioritystring" to control encryption + parameters used by GnuTLS +- imrelp: + * new parameter "compression.dhbits" to control the number of + bits being used for Diffie-Hellman key generation + * new parameter "compression.prioritystring" to control encryption + parameters used by GnuTLS + * support for impstats added + * support for setting permitted peers (client authentication) added + * bugfix: potential segfault at startup on invalid config parameters +- imjournal: imported patches from 7.4.1 +- omprog: add support for command line parameters +- added experimental TCP stream compression (imptcp only, currently) +- added BSD-specific syslog facilities + * "console" + * "bsd_security" - this is called "security" under BSD, but that name + was unfortunately already taken by some standard facility. So I + did the (hopefully) second-best thing and renamed it a little. +- imported fixes from 7.4.2 (especially build problems on FreeBSD) +- bugfix: imptcp did not properly initialize compression status variable + could lead to segfault if stream:always compression mode was selected +--------------------------------------------------------------------------- +Version 7.5.0 [devel] 2013-06-11 +- imrelp: implement "ruleset" module parameter +- imrelp/omrelp: add TLS & compression (zip) support +- omrelp: add "rebindInterval" parameter +- add -S command line option to specify IP address to use for RELP client + connections + Thanks to Axel Rau for the patch. +--------------------------------------------------------------------------- Version 7.4.3 [v7.4-stable] 2013-07-?? - bugfix: omlibdbi did not properly close connection on some errors This happened to errors occuring in Begin/End Transaction entry diff --git a/Makefile.am b/Makefile.am index 567b8769..ee4e7f31 100644 --- a/Makefile.am +++ b/Makefile.am @@ -241,6 +241,14 @@ if ENABLE_MMANON SUBDIRS += plugins/mmanon endif +if ENABLE_MMCOUNT +SUBDIRS += plugins/mmcount +endif + +if ENABLE_MMFIELDS +SUBDIRS += plugins/mmfields +endif + if ENABLE_ORACLE SUBDIRS += plugins/omoracle endif @@ -290,5 +298,5 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) # temporarily disable these checks for make distcheck 2012-09-06 rgerhards # --enable-extended-tests \ -# --enable-pgsql \ +# --enable-pgsql ACLOCAL_AMFLAGS = -I m4 @@ -357,7 +357,7 @@ finalize_it: /* action construction finalizer */ rsRetVal -actionConstructFinalize(action_t *pThis, struct cnfparamvals *queueParams) +actionConstructFinalize(action_t *pThis, struct nvlst *lst) { DEFiRet; uchar pszAName[64]; /* friendly name of our action */ @@ -432,7 +432,7 @@ actionConstructFinalize(action_t *pThis, struct cnfparamvals *queueParams) obj.SetName((obj_t*) pThis->pQueue, pszAName); qqueueSetpAction(pThis->pQueue, pThis); - if(queueParams == NULL) { /* use legacy params? */ + if(lst == NULL) { /* use legacy params? */ /* ... set some properties ... */ # define setQPROP(func, directive, data) \ CHKiRet_Hdlr(func(pThis->pQueue, data)) { \ @@ -466,7 +466,7 @@ actionConstructFinalize(action_t *pThis, struct cnfparamvals *queueParams) } else { /* we have v6-style config params */ qqueueSetDefaultsActionQueue(pThis->pQueue); - qqueueApplyCnfParam(pThis->pQueue, queueParams); + qqueueApplyCnfParam(pThis->pQueue, lst); } # undef setQPROP @@ -1788,7 +1788,7 @@ actionApplyCnfParam(action_t *pAction, struct cnfparamvals *pvals) rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, struct cnfparamvals *actParams, - struct cnfparamvals *queueParams, int bSuspended) + struct nvlst *lst, int bSuspended) { DEFiRet; int i; @@ -1881,7 +1881,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, if(bSuspended) actionSuspend(pAction); - CHKiRet(actionConstructFinalize(pAction, queueParams)); + CHKiRet(actionConstructFinalize(pAction, lst)); /* TODO: if we exit here, we have a memory leak... */ @@ -1940,7 +1940,6 @@ rsRetVal actionNewInst(struct nvlst *lst, action_t **ppAction) { struct cnfparamvals *paramvals; - struct cnfparamvals *queueParams; modInfo_t *pMod; uchar *cnfModName = NULL; omodStringRequest_t *pOMSR; @@ -1971,9 +1970,7 @@ actionNewInst(struct nvlst *lst, action_t **ppAction) FINALIZE; /* iRet is already set to error state */ } - qqueueDoCnfParams(lst, &queueParams); - - if((iRet = addAction(&pAction, pMod, pModData, pOMSR, paramvals, queueParams, + if((iRet = addAction(&pAction, pMod, pModData, pOMSR, paramvals, lst, (iRet == RS_RET_SUSPENDED)? 1 : 0)) == RS_RET_OK) { /* check if the module is compatible with select features * (currently no such features exist) */ @@ -91,7 +91,7 @@ struct action_s { /* function prototypes */ rsRetVal actionConstruct(action_t **ppThis); -rsRetVal actionConstructFinalize(action_t *pThis, struct cnfparamvals *queueParams); +rsRetVal actionConstructFinalize(action_t *pThis, struct nvlst *lst); rsRetVal actionDestruct(action_t *pThis); rsRetVal actionDbgPrint(action_t *pThis); rsRetVal actionSetGlobalResumeInterval(int iNewVal); @@ -99,7 +99,7 @@ rsRetVal actionDoAction(action_t *pAction); rsRetVal actionWriteToAction(action_t *pAction, msg_t *pMsg); rsRetVal actionCallHUPHdlr(action_t *pAction); rsRetVal actionClassInit(void); -rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, struct cnfparamvals *actParams, struct cnfparamvals *queueParams, int bSuspended); +rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, struct cnfparamvals *actParams, struct nvlst *lst, int bSuspended); rsRetVal activateActions(void); rsRetVal actionNewInst(struct nvlst *lst, action_t **ppAction); rsRetVal actionProcessCnf(struct cnfobj *o); diff --git a/build/rhel/rsyslog/rsyslog.spec b/build/rhel/rsyslog/rsyslog.spec index 4aa91df3..7bb367d4 100644 --- a/build/rhel/rsyslog/rsyslog.spec +++ b/build/rhel/rsyslog/rsyslog.spec @@ -47,6 +47,11 @@ Summary: ElasticSearch output module for rsyslog Group: System Environment/Daemons Requires: %name = %version-%release +%package mmcount +Summary: Message counting support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release + %package mmjsonparse Summary: JSON enhanced logging support Group: System Environment/Daemons @@ -154,6 +159,11 @@ relay chains. This module provides the capability for rsyslog to feed logs directly into Elasticsearch. +%description mmcount +This module provides the capability to count log messages by severity +or json property of given app-name. The count value is added into the +log message in json property named 'mmcount' + %description mmjsonparse This module provides the capability to recognize and parse JSON enhanced syslog messages. @@ -226,6 +236,7 @@ export PKG_CONFIG=/usr/bin/pkg-config %configure --disable-static \ --disable-testbench \ --enable-elasticsearch \ + --enable-mmcount \ --enable-mmjsonparse \ --enable-mmnormalize \ --enable-imzmq3 \ @@ -339,6 +350,10 @@ fi %defattr(-,root,root) %{_libdir}/rsyslog/omelasticsearch.so +%files mmcount +%defattr(-,root,root) +%{_libdir}/rsyslog/mmcount.so + %files mmjsonparse %defattr(-,root,root) %{_libdir}/rsyslog/mmjsonparse.so diff --git a/configure.ac b/configure.ac index db324c7c..f5060f73 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[7.4.2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[7.5.2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -962,6 +962,32 @@ AC_ARG_ENABLE(mmanon, AM_CONDITIONAL(ENABLE_MMANON, test x$enable_mmanon = xyes) +# mmcount +AC_ARG_ENABLE(mmcount, + [AS_HELP_STRING([--enable-mmcount],[Enable message counting @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_xmpp="yes" ;; + no) enable_xmpp="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-mmcount) ;; + esac], + [enable_mmcount=no] +) +AM_CONDITIONAL(ENABLE_MMCOUNT, test x$enable_mmcount = xyes) + + +# mmfields +AC_ARG_ENABLE(mmfields, + [AS_HELP_STRING([--enable-mmfields],[Enable building mmfields support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_mmfields="yes" ;; + no) enable_mmfields="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-mmfields) ;; + esac], + [enable_mmfields=no] +) +AM_CONDITIONAL(ENABLE_MMFIELDS, test x$enable_mmfields = xyes) + + # RELP support AC_ARG_ENABLE(relp, [AS_HELP_STRING([--enable-relp],[Enable RELP support @<:@default=no@:>@])], @@ -973,7 +999,7 @@ AC_ARG_ENABLE(relp, [enable_relp=no] ) if test "x$enable_relp" = "xyes"; then - PKG_CHECK_MODULES(RELP, relp >= 1.0.3) + PKG_CHECK_MODULES(RELP, relp >= 1.2.0) fi AM_CONDITIONAL(ENABLE_RELP, test x$enable_relp = xyes) @@ -1478,6 +1504,8 @@ AC_CONFIG_FILES([Makefile \ plugins/mmjsonparse/Makefile \ plugins/mmaudit/Makefile \ plugins/mmanon/Makefile \ + plugins/mmcount/Makefile \ + plugins/mmfields/Makefile \ plugins/omelasticsearch/Makefile \ plugins/sm_cust_bindcdr/Makefile \ plugins/mmsnmptrapd/Makefile \ @@ -1501,6 +1529,8 @@ echo " uuid support enabled: $enable_uuid" echo " Log file signing support: $enable_guardtime" echo " Log file encryption support: $enable_libgcrypt" echo " anonymization support enabled: $enable_mmanon" +echo " message counting support enabled: $enable_mmcount" +echo " mmfields enabled: $enable_mmfields" echo echo "---{ input plugins }---" echo " Klog functionality enabled: $enable_klog ($os_type)" @@ -35,7 +35,7 @@ rsRetVal multiSubmitFlush(multi_submit_t *pMultiSub); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal __attribute__((deprecated)) parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime, ruleset_t *pRuleset); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ -rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct cnfparamvals *queueParams); +rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct nvlst *lst); extern int MarkInterval; extern qqueue_t *pMsgQueue; /* the main message queue */ diff --git a/doc/build_from_repo.html b/doc/build_from_repo.html index a06863e1..693704a8 100644 --- a/doc/build_from_repo.html +++ b/doc/build_from_repo.html @@ -8,7 +8,7 @@ distribution tarball to generate it. But there may be situations where it is des to build directly from the source repository. This is useful for people who would like to participate in development or who would like to use the latest, not-yet-released code. The later may especially be the case if you are asked to try out an experimental version. -<p>Building from the repsitory is not much different than building from the source +<p>Building from the repository is not much different than building from the source tarball, but some files are missing because they are output files and thus do not belong into the repository. <h2>Obtaining the Source</h2> diff --git a/doc/debug.html b/doc/debug.html index 557ca6d3..537cd6b4 100644 --- a/doc/debug.html +++ b/doc/debug.html @@ -27,7 +27,7 @@ be replaced by something else.</p> <p>There are two environment variables that set several debug settings: <ul> <li>The "RSYSLOG_DEBUGLOG" (sample: RSYSLOG_DEBUGLOG="/path/to/debuglog/") -writes (allmost) +writes (almost) all debug message to the specified log file in addition to stdout. Some system messages (e.g. segfault or abort message) are not written to the file as we can not capture them. @@ -133,7 +133,7 @@ turned on. threads and their calling stack by sending SIGUSR2. However, the usefulness of that information is very much depending on rsyslog compile-time settings, must importantly the --enable-rtinst configure flag. Note that activating this option causes additional overhead -and slows down rsyslgod considerable. So if you do that, you need to check if it is +and slows down rsyslogd considerable. So if you do that, you need to check if it is capable to handle the workload. Also, threading behavior is modified by the runtime instrumentation. <p>Sending SIGUSR2 writes new process state information to the log file each time @@ -143,13 +143,13 @@ some diagnostic information on the current processing state. In that case, turni on the mutex debugging options (see above) is probably useful. <h2>Interpreting the Logs</h2> <p>Debug logs are primarily meant for rsyslog developers. But they may still provide valuable -information to users. Just be warned that logs sometimes contains informaton the looks like +information to users. Just be warned that logs sometimes contains information the looks like an error, but actually is none. We put a lot of extra information into the logs, and there are some cases where it is OK for an error to happen, we just wanted to record it inside the log. The code handles many cases automatically. So, in short, the log may not make sense to you, but it (hopefully) makes sense to a developer. Note that we developers often need many lines of the log file, it is relatively rare that a problem can be diagnosed by -looking at just a couple of (hundered) log records. +looking at just a couple of (hundred) log records. <h2>Security Risks</h2> <p>The debug log will reveal potentially sensible information, including user accounts and passwords, to anyone able to read the log file. As such, it is recommended to properly @@ -159,7 +159,7 @@ attack or try to hide some information from the log file. As such, it is suggest enable DebugOnDemand mode only for a reason. Note that when no debug mode is enabled, SIGUSR1 and SIGUSR2 are completely ignored. <p>When running in any of the debug modes (including on demand mode), an interactive -instance of rsyslogd can be aborted by pressing ctl-c. +instance of rsyslogd can be aborted by pressing ctrl-c. <p> <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html index b33b67f9..802de804 100644 --- a/doc/dev_oplugins.html +++ b/doc/dev_oplugins.html @@ -15,7 +15,7 @@ and pointers than to have nothing. <p>The best to get started with rsyslog plugin development is by looking at existing plugins. All that start with "om" are <b>o</b>utput <b>m</b>odules. That means they are primarily thought of being message sinks. In theory, however, output -plugins may aggergate other functionality, too. Nobody has taken this route so far +plugins may aggregate other functionality, too. Nobody has taken this route so far so if you would like to do that, it is highly suggested to post your plan on the rsyslog mailing list, first (so that we can offer advise). <p>The rsyslog distribution tarball contains two plugins that are extremely well @@ -29,7 +29,7 @@ plugins. It is bare of real functionality but has ample comments. Even if you de to start from another plugin (or even from scratch), be sure to read omtemplate source and comments first. The omstdout is primarily a testing aide, but offers support for the two different parameter-passing conventions plugins can use (plus the way to -differentiate between the two). It also is not bare of functionaly, only mostly +differentiate between the two). It also is not bare of functionality, only mostly bare of it ;). But you can actually execute it and play with it. <p>In any case, you should also read the comments in ./runtime/module-template.h. Output plugins are build based on a large set of code-generating macros. These @@ -65,7 +65,7 @@ that shares a single instanceData structure. <p>So as long as you do not mess around with global data, you do not need to think about multithreading (and can apply a purely sequential programming methodology). -<p>Please note that duringt the configuraton parsing stage of execution, access to +<p>Please note that during the configuration parsing stage of execution, access to global variables for the configuration system is safe. In that stage, the core will only call sequentially into the plugin. <h3>Getting Message Data</h3> @@ -90,7 +90,7 @@ get it into the core (so far, we could accept all such suggestions - no promise, request access to the template components. The typical use case seems to be databases, where you would like to access properties via specific fields. With that mode, you receive a char ** array, where each array element points to one field from the template (from -left to right). Fields start at arrray index 0 and a NULL pointer means you have +left to right). Fields start at array index 0 and a NULL pointer means you have reached the end of the array (the typical Unix "poor man's linked list in an array" design). Note, however, that each of the individual components is a string. It is not a date stamp, number or whatever, but a string. This is because rsyslog processes @@ -153,19 +153,19 @@ for example in MongoDB or ElasticSearch. a single-message interface was supported. <p>With the <b>single message</b> plugin interface, each message is passed via a separate call to the plugin. Most importantly, the rsyslog engine assumes that each call to the plugin is a complete transaction -and as such assumes that messages be properly commited after the plugin returns to the engine. +and as such assumes that messages be properly committed after the plugin returns to the engine. <p>With the <b>batching</b> interface, rsyslog employs something along the line of "transactions". Obviously, the rsyslog core can not make non-transactional outputs to be fully transactional. But what it can is support that the output tells the core which -messages have been commited by the output and which not yet. The core can than take care -of those uncommited messages when problems occur. For example, if a plugin has received -50 messages but not yet told the core that it commited them, and then returns an error state, the +messages have been committed by the output and which not yet. The core can than take care +of those uncommitted messages when problems occur. For example, if a plugin has received +50 messages but not yet told the core that it committed them, and then returns an error state, the core assumes that all these 50 messages were <b>not</b> written to the output. The core then -requeues all 50 messages and does the usual retry processing. Once the output plugin tells the +re-queues all 50 messages and does the usual retry processing. Once the output plugin tells the core that it is ready again to accept messages, the rsyslog core will provide it with these 50 -not yet commited messages again (actually, at this point, the rsyslog core no longer knows that -it is re-submiting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 -messages were commited (before it failed), then only 10 would have been requeued and resubmitted. +not yet committed messages again (actually, at this point, the rsyslog core no longer knows that +it is re-submitting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 +messages were committed (before it failed), then only 10 would have been re-queued and resubmitted. <p>In order to provide an efficient implementation, there are some (mild) constraints in that transactional model: first of all, rsyslog itself specifies the ultimate transaction boundaries. That is, it tells the plugin when a transaction begins and when it must finish. The plugin @@ -176,7 +176,7 @@ transaction support. Note that batch sizes are variable within the range of 1 to maximum limit. Most importantly, that means that plugins may receive batches of single messages, so they are required to commit each message individually. If the plugin tries to be "smarter" than the rsyslog engine and does not commit messages in those cases (for example), the plugin -puts message stream integrity at risk: once rsyslog has notified the plugin of transacton end, +puts message stream integrity at risk: once rsyslog has notified the plugin of transaction end, it discards all messages as it considers them committed and save. If now something goes wrong, the rsyslog core does not try to recover lost messages (and keep in mind that "goes wrong" includes such uncontrollable things like connection loss to a database server). So it is @@ -191,8 +191,8 @@ This is also under evaluation and, once decided, the core will offer an interfac to preserve message stream integrity for properly-crafted plugins). <p>The second restriction is that if a plugin makes commits in between (what is perfectly legal) those commits must be in-order. So if a commit is made for message ten out of 50, -this means that messages one to nine are also commited. It would be possible to remove -this restriction, but we have decided to deliberately introduce it to simpify things. +this means that messages one to nine are also committed. It would be possible to remove +this restriction, but we have decided to deliberately introduce it to simplify things. <h3>Output Plugin Transaction Interface</h3> <p>In order to keep compatible with existing output plugins (and because it introduces no complexity), the transactional plugin interface is build on the traditional @@ -252,7 +252,7 @@ But they convey additional information about the commit status as follows: <table border="0"> <tr> <td valign="top"><i>RS_RET_OK</i></td> -<td>The record and all previous inside the batch has been commited. +<td>The record and all previous inside the batch has been committed. <i>Note:</i> this definition is what makes integrating plugins without the transaction being/end calls so easy - this is the traditional "success" return state and if every call returns it, there is no need for actually calling @@ -260,7 +260,7 @@ state and if every call returns it, there is no need for actually calling </tr> <tr> <td valign="top"><i>RS_RET_DEFER_COMMIT</i></td> -<td>The record has been processed, but is not yet commited. This is the +<td>The record has been processed, but is not yet committed. This is the expected state for transactional-aware plugins.</td> </tr> <tr> @@ -269,7 +269,7 @@ expected state for transactional-aware plugins.</td> current one not yet. This state is introduced to support sources that fill up buffers and commit once a buffer is completely filled. That may occur halfway in the next record, so it may be important to be able to tell the -engine the everything up to the previouos record is commited</td> +engine the everything up to the previous record is committed</td> </tr> </table> <p>Note that the typical <b>calling cycle</b> is <code>beginTransaction()</code>, @@ -290,7 +290,7 @@ exists. So we introduce it with that release. What the means is if a rsyslog cor not provide this query interface, it is a core that was build before batching support was available. So the absence of a query interface indicates that the transactional interface is not available. One might now be tempted the think there is no need to do -the actual check, but is is recommended to ask the rsyslog engine explicitely if +the actual check, but is is recommended to ask the rsyslog engine explicitly if the transactional interface is present and will be honored. This enables us to create versions in the future which have, for whatever reason we do not yet know, no support for this interface. diff --git a/doc/dev_queue.html b/doc/dev_queue.html index bf2af7f0..6d5fe73f 100644 --- a/doc/dev_queue.html +++ b/doc/dev_queue.html @@ -80,7 +80,7 @@ and terminating while waiting on the primary queue to fill. In practice, this is highly unlikely (but only for the main message queue) because rsyslog issues a startup message. HOWEVER, we can not rely on that, it would introduce a race. If the primary rsyslog thread (the one that issues the message) is scheduled very -late and there is a low inactivty timeout for queue workers, the queue worker +late and there is a low inactivity timeout for queue workers, the queue worker may terminate before the startup message is issued. And if the on-disk queue holds only a few messages, it may become empty before the DA worker is re-initiated again. So it is possible that the DA run mode termination criteria @@ -105,7 +105,7 @@ clean shutdown of the DA queue).</p> <p>One might think that it would be more natural for the DA queue to detect being idle and shut down itself. However, there are some issues associated with that. Most importantly, all queue worker threads need to be shut down during -queue destruction. Only after that has happend, final destruction steps can +queue destruction. Only after that has happened, final destruction steps can happen (else we would have a myriad of races). However, it is the DA queues worker thread that detects it is empty (empty queue detection always happens at the consumer side and must so). That would lead to the DA queue worker thread to @@ -115,7 +115,7 @@ destructed). Obviously, this does not work out (and I didn't even mention the other issues - so let's forget about it). As such, the thread that enqueues messages must destruct the queue - and that is the primary queue's DA worker thread.</p> -<p>There are some subleties due to thread synchronization and the fact that the +<p>There are some subtleties due to thread synchronization and the fact that the DA consumer may not be running (in a <b>case-2 startup</b>). So it is not trivial to reliably change the queue back from DA run mode to regular run mode. The priority is a clean switch. We accept the fact that there may be situations @@ -132,7 +132,7 @@ most probably even lead to worse performance under regular conditions).</p> </ol> <p>Case 2 is unlikely, but may happen (see info above on a case 2 startup).</p> <p><b>The DA worker may also not wait at all,</b> because it is actively -executing and shuffeling messages between the queues. In that case, however, the +executing and shuffling messages between the queues. In that case, however, the program flow passes both of the two wait conditions but simply does not wait.</p> <p><b>Finally, the DA worker may be inactive </b>(again, with a case-2 startup). In that case no work(er) at all is executed. Most importantly, without the DA @@ -247,4 +247,4 @@ no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at <a href="http://www.gnu.org/copyleft/fdl.html"> http://www.gnu.org/copyleft/fdl.html</a>.</p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/droppriv.html b/doc/droppriv.html index 7293e872..75773e64 100644 --- a/doc/droppriv.html +++ b/doc/droppriv.html @@ -18,7 +18,7 @@ rsyslogd needs to start up as root. user. That is probably the safest way of operations. However, if a startup as root is required, you can use the $PrivDropToGroup and $PrivDropToUser config directives to specify a group and/or user that rsyslogd should drop to after initialization. -Once this happend, the daemon runs without high privileges (depending, of +Once this happens, the daemon runs without high privileges (depending, of course, on the permissions of the user account you specified). <p>There is some additional information available in the <a href="http://wiki.rsyslog.com/index.php/Security#Dropping_Privileges">rsyslog wiki</a>. diff --git a/doc/features.html b/doc/features.html index 626ff65d..ecf6a014 100644 --- a/doc/features.html +++ b/doc/features.html @@ -42,7 +42,7 @@ into syslog messages (one per line)</li> <li>ability to configure backup syslog/database servers - if the primary fails, control is switched to a prioritized list of backups</li> <li>support for receiving messages via reliable <a href="http://www.monitorware.com/Common/en/glossary/rfc3195.php"> -RFC 3195</a> delivery (a bit clumpsy to build right now...)</li> +RFC 3195</a> delivery (a bit clumsy to build right now...)</li> <li>ability to generate file names and directories (log targets) dynamically, based on many different properties</li> <li>control of log output format, including ability to present @@ -95,7 +95,7 @@ via custom plugins</li> <li> an easy-to-write to plugin interface</li> <li> ability to send SNMP trap messages</li> <li> ability to filter out messages based on sequence of arrival</li> -<li>support for comma-seperated-values (CSV) output generation +<li>support for comma-separated-values (CSV) output generation (via the "csv" property replace option). The CSV format supported is that from RFC 4180.</li> <li>support for arbitrary complex boolean, string and diff --git a/doc/free_support.html b/doc/free_support.html index 182a82cd..a3a9a69a 100644 --- a/doc/free_support.html +++ b/doc/free_support.html @@ -18,7 +18,7 @@ reason is quite simple: If I do personal support, you gain some advantage withou contributing something back. Think about it: if you ask your question on the public forum or mailing list, other with the same problem can you and, most importantly, even years later find your post (and the answer) and get the problem solved. So by -solving your issue in public, you help create a great community ressource and also +solving your issue in public, you help create a great community resource and also help your fellow users finding solutions quicker. In the long term, this also contributes to improved code because the more questions users can find solutions to themselves, the fewer I need to look at. diff --git a/doc/history.html b/doc/history.html index 57b64004..a071461e 100644 --- a/doc/history.html +++ b/doc/history.html @@ -117,7 +117,7 @@ the need to have complex expression support, which was also the first use case. On February, 28th rsyslog 3.12.0 was released, the first version to contain expression support. This also meant that rsyslog from that date on supported all syslog-ng major features, but had a -number of major features exlusive to it. With 3.12.0, I consider +number of major features exclusive to it. With 3.12.0, I consider rsyslog fully superior to syslog-ng (except for platform support).</p> <p>Following the Fedora Developer's conference in Brno <b>2012</b>, rsyslog diff --git a/doc/im3195.html b/doc/im3195.html index aad9f3d1..317ab840 100644 --- a/doc/im3195.html +++ b/doc/im3195.html @@ -25,7 +25,7 @@ The port on which imklog listens for RFC 3195 messages. The default port is 601 to this input module, but we have NOT conducted any testing. Also, the module does not yet properly handle the recovery case. If someone intends to put this module into production, good testing should be -cunducted. It also is a good idea to notify the rsyslog project that you intend to use +conducted. It also is a good idea to notify the rsyslog project that you intend to use it in production. In this case, we'll probably give the module another cleanup. We don't do this now because so far it looks just like a big waste of time. diff --git a/doc/imklog.html b/doc/imklog.html index 1f195b16..f2e36fd2 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -25,7 +25,7 @@ imklog generates some messages of itself (e.g. on problems, startup and shutdown) and these do not stem from the kernel. Historically, under Linux, these too have "kern" facility. Thus, on Linux platforms the default is "kern" while on others it is "syslogd". You usually do not -need to specify this configuratin directive - it is included primarily +need to specify this configuration directive - it is included primarily for few limited cases where it is needed for good reason. Bottom line: if you don't have a good idea why you should use this setting, do not touch it.</li> diff --git a/doc/imkmsg.html b/doc/imkmsg.html index 23b96147..ba73715d 100644 --- a/doc/imkmsg.html +++ b/doc/imkmsg.html @@ -16,7 +16,7 @@ Milan Bartos <p>Reads messages from the /dev/kmsg structured kernel log and submits them to the syslog engine.</p> <p> -The printk log buffer constains log records. These records are exported by /dev/kmsg +The printk log buffer contains log records. These records are exported by /dev/kmsg device as structured data in the following format:<br /> "level,sequnum,timestamp;<message text>\n"<br /> There could be continuation lines starting with space that contains key/value pairs.<br /> diff --git a/doc/impstats.html b/doc/impstats.html index 8db9c6f6..1da09ced 100644 --- a/doc/impstats.html +++ b/doc/impstats.html @@ -12,7 +12,7 @@ <p><b>Description</b>:</p> <p>This module provides periodic output of rsyslog internal counters. Note that the whole statistics system is currently under development. So -availabilty and format of counters may change and is not yet stable (so be +availability and format of counters may change and is not yet stable (so be prepared to change your trending scripts when you upgrade to a newer rsyslog version). <p>The set of available counters will be output as a set of syslog messages. This output is periodic, with the interval being configurable (default is 5 minutes). @@ -20,7 +20,7 @@ Be sure that your configuration records the counter messages (default is syslog. Besides logging to the regular syslog stream, the module can also be configured to write statistics data into a (local) file. <p>Note that loading this module has impact on rsyslog performance. Depending on -settings, this impact may be noticable (for high-load environments). +settings, this impact may be noticeable (for high-load environments). <p>The rsyslog website has an updated overview of available <a href="http://rsyslog.com/rsyslog-statistic-counter/">rsyslog statistic counters</a>. </p> diff --git a/doc/imptcp.html b/doc/imptcp.html index aece428d..41283e69 100644 --- a/doc/imptcp.html +++ b/doc/imptcp.html @@ -77,7 +77,7 @@ The default, 0, means that the operating system defaults are used. This has only effect if keep-alive is enabled. The functionality may not be available on all platforms. <li><b>KeepAlive.Interval</b> <number><br> -The interval between subsequential keepalive probes, regardless of what the connection has exchanged in the meantime. +The interval between subsequent keepalive probes, regardless of what the connection has exchanged in the meantime. The default, 0, means that the operating system defaults are used. This has only effect if keep-alive is enabled. The functionality may not be available on all platforms. @@ -102,6 +102,11 @@ of seconds (5 recommended) to activate rate-limiting. </li> <li><b>RateLimit.Burst</b> [number] - (available since 7.3.1) specifies the rate-limiting burst in number of messages. Default is 10,000. +<li><b>compression.mode</b><i>mode</i><br> +<i>mode</i> is one of "none" or "stream:always". +It is the counterpart to the compression modes set in +<a href="omfile.html">omfile</a>. +Please see it's documentation for details. </li> </ul> <b>Caveats/Known Bugs:</b> diff --git a/doc/imrelp.html b/doc/imrelp.html index f7fcc4b3..66e8b054 100644 --- a/doc/imrelp.html +++ b/doc/imrelp.html @@ -28,14 +28,84 @@ nits outlined above, is a much more reliable solution than plain tcp syslog and so it is highly suggested to use RELP instead of plain tcp. Clients send messages to the RELP server via omrelp.</p> -<p><b>Configuration Directives</b>:</p> +<p><b>Module Parameters</b>:</p> +<ul> + <li><b>Ruleset</b> <name></br> + Binds the specified ruleset to <b>all</b> RELP listeners. +</ul> +<p><b>Input Parameters</b>:</p> <ul> <li><b>Port</b> <port><br> Starts a RELP server on selected port</li> +<li><b>tls</b> (not mandatory, values "on","off", default "off")<br> +If set to "on", the RELP connection will be encrypted by TLS, +so that the data is protected against observers. Please note +that both the client and the server must have set TLS to +either "on" or "off". Other combinations lead to unpredictable +results. +</li> +<li><b>tls.compression</b> (not mandatory, values "on","off", default "off")<br> +The controls if the TLS stream should be compressed (zipped). While this +increases CPU use, the network bandwidth should be reduced. Note that +typical text-based log records usually compress rather well. +</li> +<li><b>tls.dhbits</b> (not mandatory, integer)<br> +This setting controls how many bits are used for Diffie-Hellman key +generation. If not set, the librelp default is used. For secrity +reasons, at least 1024 bits should be used. Please note that the number +of bits must be supported by GnuTLS. If an invalid number is given, rsyslog +will report an error when the listener is started. We do this to be transparent +to changes/upgrades in GnuTLS (to check at config processing time, we would need +to hardcode the supported bits and keep them in sync with GnuTLS - this is +even impossible when custom GnuTLS changes are made...). +</li> +<li><b>tls.permittedPeer</b> peer</br> +Places access restrictions on this listener. Only peers which +have been listed in this parameter may connect. The validation +bases on the certificate the remote peer presents.<br> +The <i>peer</i> parameter lists permitted certificate +fingerprints. Note that it is an array parameter, so either +a single or multiple fingerprints can be listed. When a +non-permitted peer connects, the refusal is logged together +with it's fingerprint. So if the administrator knows this was +a valid request, he can simple add the fingerprint by copy and +paste from the logfile to rsyslog.conf. +<br>To specify multiple fingerprints, just enclose them +in braces like this: +<br>tls.permittedPeer=["SHA1:...1", "SHA1:....2"] +<br>To specify just a single peer, you can either +specify the string directly or enclose it in braces. +</li> +<li><b>tls.authMode</b> mode</br> +Sets the mode used for mutual authentication. Supported values are +either "<i>fingerprint</i>" or "<i>name"</i>. +<br>Fingerprint mode basically is what SSH +does. It does not require a full PKI to be present, instead self-signed +certs can be used on all peers. Even if a CA certificate is given, the +validity of the peer cert is NOT verified against it. Only the +certificate fingerprint counts. +<br>In "name" mode, certificate validation happens. Here, the matching +is done against the certificate's subjectAltName and, as a fallback, +the subject common name. If the certificate contains multiple names, +a match on any one of these names is considered good and permits the +peer to talk to rsyslog. +<li><b>tls.prioritystring</b> (not mandatory, string)<br> +This parameter permits to specify the so-called "priority string" to +GnuTLS. This string gives complete control over all crypto parameters, +including compression setting. For this reason, when the prioritystring +is specified, the "tls.compression" parameter has no effect and is +ignored. +<br>Full information about how to construct a priority string can be +found in the GnuTLS manual. At the time of this writing, this +information was contained in +<a href="http://gnutls.org/manual/html_node/Priority-Strings.html">section 6.10 of the GnuTLS manual</a>. +<br><b>Note: this is an expert parameter.</b> Do not use if you do +not exactly know what you are doing. +</li> </ul> <b>Caveats/Known Bugs:</b> <ul> -<li>ruleset can only be bound via legacy configuration format</li> +<li>see description</li> <li>To obtain the remote system's IP address, you need to have at least librelp 1.0.0 installed. Versions below it return the hostname instead of the IP address.</li> @@ -45,20 +115,19 @@ not specific ones. This is due to a currently existing limitation in librelp. <p><b>Sample:</b></p> <p>This sets up a RELP server on port 20514.<br> </p> -<textarea rows="15" cols="60">module(load="imrelp") # needs to be done just once +<textarea rows="5" cols="60">module(load="imrelp") # needs to be done just once input(type="imrelp" port="20514") </textarea> <p><b>Legacy Configuration Directives</b>:</p> <ul> <li>InputRELPServerBindRuleset <name> (available in 6.3.6+)</br> -Binds the specified ruleset to all RELP listeners. +equivalent to: RuleSet <li>InputRELPServerRun <port><br> equivalent to: Port</li> </ul> <b>Caveats/Known Bugs:</b> <ul> -<li>see description</li> <li>To obtain the remote system's IP address, you need to have at least librelp 1.0.0 installed. Versions below it return the hostname instead of the IP address.</li> @@ -68,14 +137,14 @@ not specific ones. This is due to a currently existing limitation in librelp. <p><b>Sample:</b></p> <p>This sets up a RELP server on port 20514.<br> </p> -<textarea rows="15" cols="60">$ModLoad imrelp # needs to be done just once +<textarea rows="5" cols="60">$ModLoad imrelp # needs to be done just once $InputRELPServerRun 20514 </textarea> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/imsolaris.html b/doc/imsolaris.html index ce0e7e84..cc9a745f 100644 --- a/doc/imsolaris.html +++ b/doc/imsolaris.html @@ -18,7 +18,7 @@ is no special kernel input device. Instead, both kernel messages as well as messages emitted via syslog() are received from a single source. <p>This module obeys the Solaris door() mechanism to detect a running syslogd instance. As such, only one can be active at one time. If it detects another -active intance at startup, the module disables itself, but rsyslog will +active instance at startup, the module disables itself, but rsyslog will continue to run. <p><b>Configuration Directives</b>:</p> <ul> diff --git a/doc/imtcp.html b/doc/imtcp.html index b9f0b056..5ac30d08 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -63,13 +63,13 @@ the sender is throttled a bit when the queue becomes near-full. This is done in to preserve some queue space for inputs that can not throttle (like UDP), but it may have some undesired effect in some configurations. Still, we consider this as a useful setting and thus it is the default. To turn the handling off, simply -configure that explicitely. +configure that explicitly. </li> <li><b>MaxListeners</b> <number><br> Sets the maximum number of listeners (server ports) supported. Default is 20. This must be set before the first $InputTCPServerRun directive.</li> <li><b>MaxSessions</b> <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> <li><b>StreamDriver.Mode</b> <number><br> -Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specifc.</li> +Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specific.</li> <li><b>StreamDriver.AuthMode</b> <mode-string><br> Sets the authentication mode for the currently selected <a href="netstream.html">network stream driver</a>. <mode-string> is driver specifc.</li> <li><b>PermittedPeer</b> <id-string><br> diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 0affe8c3..80c3bc5a 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -42,7 +42,7 @@ severity and configure things accordingly. To turn off rate limiting, set the interval to zero. <p><b>Unix log sockets can be flow-controlled.</b> That is, if processing queues fill up, the unix socket reader is blocked for a short while. This may be useful to prevent overruning -the queues (which may cause exessive disk-io where it actually would not be needed). However, +the queues (which may cause excessive disk-io where it actually would not be needed). However, flow-controlling a log socket (and especially the system log socket) can lead to a very unresponsive system. As such, flow control is disabled by default. That means any log records are places as quickly as possible into the processing queues. If you would like to have @@ -124,7 +124,7 @@ module documentation for a more in-depth description. to the next socket.</li> <li><b>RateLimit.Interval</b> [number] - specifies the rate-limiting interval in seconds. Default value is 0, which turns off rate limiting. Set it to a number -of seconds (5 recommended) to activate rate-limiting. The default of 0 has been choosen +of seconds (5 recommended) to activate rate-limiting. The default of 0 has been chosen as people experienced problems with this feature activated by default. Now it needs an explicit opt-in by setting this parameter. </li> @@ -144,7 +144,7 @@ be obtained from the log socket itself. If so, the TAG part of the message is re It is recommended to turn this option on, but the default is "off" to keep compatible with earlier versions of rsyslog. </li> <li><b>UseSysTimeStamp</b> [<b>on</b>/off] instructs imuxsock -to obtain message time from the system (via control messages) insted of using time +to obtain message time from the system (via control messages) instead of using time recorded inside the message. This may be most useful in combination with systemd. Note: this option was introduced with version 5.9.1. Due to the usefulness of it, we decided to enable it by default. As such, 5.9.1 and above behave slightly different diff --git a/doc/licensing.html b/doc/licensing.html index 93a50930..0a4ab1f4 100644 --- a/doc/licensing.html +++ b/doc/licensing.html @@ -22,7 +22,7 @@ real details, check source files and the files COPYING and COPYING.LESSER inside <p>Each of these components can be thought of as individual projects. In fact, some of the plugins have different main authors than the rest of the rsyslog package. All of these components are currently put together into a single "rsyslog" package (tarball) for -convinience: this makes it easier to distribute a consistent version where everything +convenience: this makes it easier to distribute a consistent version where everything is included (and in the right versions) to build a full system. Platform package maintainers in general take the overall package and split off the individual components, so that users can install only what they need. In source installations, this can be done via the diff --git a/doc/log_rotation_fix_size.html b/doc/log_rotation_fix_size.html index 51edf033..7cdecc6c 100644 --- a/doc/log_rotation_fix_size.html +++ b/doc/log_rotation_fix_size.html @@ -31,7 +31,7 @@ Channels to achieve this. Putting the following directive</p> <p><pre> # start log rotation via outchannel -# outchannel definiation +# outchannel definition $outchannel log_rotation,/var/log/log_rotation.log, 52428800,/home/me/./log_rotation_script # activate the channel and log everything to it *.* :omfile:$log_rotation @@ -49,13 +49,13 @@ mv -f /var/log/log_rotation.log /var/log/log_rotation.log.1 <p>This moves the original log to a kind of backup log file. After the action was successfully performed rsyslog creates a new /var/log/log_rotation.log -file and fill it up with new logs. So the latest logs are always in log_roatation.log.</p> +file and fill it up with new logs. So the latest logs are always in log_rotation.log.</p> <h2>Conclusion</h2> <p>With this approach two files for logging are used, each with a maximum size of 50 MB. So we can say we have successfully configured a log rotation which satisfies our requirement. -We keep the logs at a fixed-size level of100 MB.</p> +We keep the logs at a fixed-size level of 100 MB.</p> <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> diff --git a/doc/manual.html b/doc/manual.html index 3c0c6ce3..8d5eb733 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ professional services</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 7.4.2 (v7.4-stable branch) of rsyslog.</b> +<p><b>This documentation is for version 7.5.2 (devel branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might diff --git a/doc/messageparser.html b/doc/messageparser.html index 370db59f..d22021dd 100644 --- a/doc/messageparser.html +++ b/doc/messageparser.html @@ -23,7 +23,7 @@ the rsyslog code). like input and output modules). That means that new message parsers can be added without modifying the rsyslog core, even without contributing something back to the project. -<p>But that doesn't answer what a message parser really is. What does ist mean to "parse a +<p>But that doesn't answer what a message parser really is. What does it mean to "parse a message" and, maybe more importantly, what is a message? To answer these questions correctly, we need to dig down into the relevant standards. <a href="http://tools.ietf.org/html/rfc5424">RFC5424</a> specifies a layered architecture @@ -49,7 +49,7 @@ reason) a single message into two and encapsulates these into two frames, there a message parser could undo that. <p>A typical example may be a multi-line message: let's assume some originator has generated a message for the format "A\nB" (where \n means LF). If that message is being transmitted -via plain tcp syslog, the frame delimiter is LF. So the sender will delimite the frame with +via plain tcp syslog, the frame delimiter is LF. So the sender will delimit the frame with LF, but otherwise send the message unmodified onto the wire (because that is how things are -unfortunately- done in plain tcp syslog...). So wire will see "A\nB\n". When this arrives at the receiver, the transport layer will undo the framing. When it sees the LF @@ -58,7 +58,7 @@ the receive will extract one complete message A and one complete message B, not that they once were both part of a large multi-line message. These two messages are then passed to the upper layers, where the message parsers receive them and extract information. However, the message parsers never know (or even have a chance to see) that A and B -belonged together. Even further, in rsyslog there is no guarnatee that A will be parsed +belonged together. Even further, in rsyslog there is no guarantee that A will be parsed before B - concurrent operations may cause the reverse order (and do so very validly). <p>The important lesson is: <b>message parsers can not be used to fix a broken framing</b>. You need a full protocol implementation to do that, what is the domain of input and @@ -73,10 +73,10 @@ the real-world evil that you can usually see. So I won't repeat that here. But i real problem is not the framing, but how to make malformed messages well-looking. <p><b>This is what message parsers permit you to do: take a (well-known) malformed message, parse it according to its semantics and generate perfectly valid internal message representations -from it.</b> So as long as messages are consistenly in the same wrong format (and they usually +from it.</b> So as long as messages are consistently in the same wrong format (and they usually are!), a message parser can look at that format, parse it, and make the message processable just -like it were wellformed in the first place. Plus, one can abuse the interface to do some other -"intersting" tricks, but that would take us to far. +like it were well formed in the first place. Plus, one can abuse the interface to do some other +"interesting" tricks, but that would take us to far. <p>While this functionality may not sound exciting, it actually solves a very big issue (that you only really understand if you have managed a system with various different syslog sources). Note that we were often able to process malformed messages in the past with the help of the @@ -113,15 +113,15 @@ interface probably extended, to support generic filter modules. These would need to the root of the parser chain. As mentioned, the current system already supports this. <p>The position inside the parser chain can be thought of as a priority: parser sitting earlier in the chain take precedence over those sitting later in it. So more specific -parser should go ealier in the chain. A good example of how this works is the default parser +parser should go earlier in the chain. A good example of how this works is the default parser set provided by rsyslog: rsyslog.rfc5424 and rsyslog.rfc3164, each one parses according to the rfc that has named it. RFC5424 was designed to be distinguishable from RFC3164 message by the sequence "1 " immediately after the so-called PRI-part (don't worry about these words, it is -sufficient if you understand there is a well-defined sequence used to indentify RFC5424 +sufficient if you understand there is a well-defined sequence used to identify RFC5424 messages). In contrary, RFC3164 actually permits everything as a valid message. Thus the RFC3164 parser will always parse a message, sometimes with quite unexpected outcome (there is a lot of guesswork involved in that parser, which unfortunately is unavoidable due to -existing techology limits). So the default parser chain is to try the RFC5424 parser first +existing technology limits). So the default parser chain is to try the RFC5424 parser first and after it the RFC3164 parser. If we have a 5424-formatted message, that parser will identify and parse it and the rsyslog engine will stop processing. But if we receive a legacy syslog message, the RFC5424 will detect that it can not parse it, return this status @@ -139,16 +139,16 @@ case, rsyslog has no other choice than to discard the message. If it does so, it a warning message, but only in the first 1,000 incidents. This limit is a safety measure against message-loops, which otherwise could quickly result from a parser chain misconfiguration. <b>If you do not tolerate loss of unparsable messages, you must ensure -that each message can be parsed.</b> You can easily achive this by always using the +that each message can be parsed.</b> You can easily achieve this by always using the "rsyslog-rfc3164" parser as the <i>last</i> parser inside parser chains. That may result in invalid parsing, but you will have a chance to see the invalid message (in debug mode, a warning message will be written to the debug log each time a message is dropped due to inability to parse it). <h3>Where are parser chains used?</h3> <p>We now know what parser chains are and how they operate. The question is now how many -parser chains can be active and how it is decicded which parser chain is used on which message. +parser chains can be active and how it is decided which parser chain is used on which message. This is controlled via <a href="multi_ruleset.html">rsyslog's rulesets</a>. In short, multiple -rulesets can be defined and there always exist at least one ruleset (for specifcs, follow +rulesets can be defined and there always exist at least one ruleset (for specifics, follow the <a href="multi_ruleset.html">link</a>). A parser chain is bound to a specific ruleset. This is done by virtue of defining parsers via the <a href="rsconf1_rulesetparser.html">$RulesetParser</a> configuration directive (for specifics, @@ -161,22 +161,22 @@ is added to the end of the (initially empty) ruleset's parser chain. <p>The correct answer is: generally yes, but it depends. First of all, remember that input modules (and specific listeners) may be bound to specific rulesets. As parser chains "reside" in rulesets, binding to a ruleset also binds to the parser chain that is bound to that ruleset. -As a number one prequisite, the input module must support binding to different rulesets. Not +As a number one prerequisite, the input module must support binding to different rulesets. Not all do, but their number is growing. For example, the important <a href="imudp.html">imudp</a> and <a href="imtcp.html">imtcp</a> input modules support that functionality. Those that do not (for example <a href="im3195">im3195</a>) can only utilize the default ruleset and thus the parser chain defined in that ruleset. <p>If you do not know if the input module in question supports ruleset binding, check -its documentation page. Those that support it have the requiered directives. +its documentation page. Those that support it have the required directives. <p>Note that it is currently under evaluation if rsyslog will support binding parser chains to specific inputs directly, without depending on the ruleset. There are some concerns that this may not be necessary but adds considerable complexity to the configuration. So this may or may not be possible in the future. In any case, if we decide to add it, input modules need to support it, so this functionality would require some time to implement. -<p>The coockbook recipe for using different parsers for different devices is given +<p>The cookbook recipe for using different parsers for different devices is given as an actual in-depth example in the <a href="rscon1_rulesetsparser.html">$RulesetParser</a> -configuration directive doc page. In short, it is acomplished by defining specific rulesets -for the required parser chains, definining different listener ports for each of the devices +configuration directive doc page. In short, it is accomplished by defining specific rulesets +for the required parser chains, defining different listener ports for each of the devices with different format and binding these listeners to the correct ruleset (and thus parser chains). Using that approach, a variety of different message formats can be supported via a single rsyslog instance. @@ -185,19 +185,19 @@ via a single rsyslog instance. <p>As of this writing, there exist only two message parsers, one for RFC5424 format and one for legacy syslog (loosely described in <a href="http://tools.ietf.org/html/rfc3164">RFC3164</a>). These parsers are built-in and -must not be explicitely loaded. However, message parsers can be added with relative ease +must not be explicitly loaded. However, message parsers can be added with relative ease by anyone knowing to code in C. Then, they can be loaded via $ModLoad just like any other loadable module. It is expected that the rsyslog project will be contributed additional message parsers over time, so that at some point there hopefully is a rich choice of them (I intend to add a browsable repository as soon as new parsers pop up). <h3>How to write a message parser?</h3> -<p>As a prequisite, you need to know the exact format that the device is sending. Then, you need +<p>As a prerequisite, you need to know the exact format that the device is sending. Then, you need moderate C coding skills, and a little bit of rsyslog internals. I guess the rsyslog specific part should not be that hard, as almost all information can be gained from the existing parsers. They are rather simple in structure and can be found under the "./tools" directory. They are named pmrfc3164.c and pmrfc5424.c. You need to follow the usual loadable module guidelines. It is my expectation that writing a parser should typically not take longer than a single -day, with maybe a day more to get aquainted with rsyslog. Of course, I am not sure if the number +day, with maybe a day more to get acquainted with rsyslog. Of course, I am not sure if the number is actually right. <p>If you can not program or have no time to do it, Adiscon can also write a message parser for you as @@ -209,7 +209,7 @@ provide a fast and efficient solution for this problem. Different parsers can be different devices, and they all convert message information into rsyslog's well-defined internal format. Message parsers were first introduced in rsyslog 5.3.4 and also offer some interesting ideas that may be explored in the future - up to full message normalization -capabilities. It is strongly recommended that anyone with a heterogenous environment take +capabilities. It is strongly recommended that anyone with a heterogeneous environment take a look at message parser capabilities. <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual diff --git a/doc/mmcount.html b/doc/mmcount.html new file mode 100644 index 00000000..1d06340d --- /dev/null +++ b/doc/mmcount.html @@ -0,0 +1,58 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>mmcount</title></head> + +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>mmcount</h1> +<p><b>Module Name: mmcount</b></p> +<p><b>Author: </b>Bala.FA <barumuga@redhat.com></p> +<p><b>Status: </b>Non project-supported module - contact author +or rsyslog mailing list for questions +<p><b>Available since</b>: 7.5.0</p> +<p><b>Description</b>:</p> +<p> +<pre> + mmcount: message modification plugin which counts messages + + This module provides the capability to count log messages by severity + or json property of given app-name. The count value is added into the + log message as json property named 'mmcount' + + Example usage of the module in the configuration file + + module(load="mmcount") + + # count each severity of appname gluster + action(type="mmcount" appname="gluster") + + # count each value of gf_code of appname gluster + action(type="mmcount" appname="glusterd" key="!gf_code") + + # count value 9999 of gf_code of appname gluster + action(type="mmcount" appname="glusterfsd" key="!gf_code" value="9999") + + # send email for every 50th mmcount + if $app-name == 'glusterfsd' and $!mmcount <> 0 and $!mmcount % 50 == 0 then { + $ActionMailSMTPServer smtp.example.com + $ActionMailFrom rsyslog@example.com + $ActionMailTo glusteradmin@example.com + $template mailSubject,"50th message of gf_code=9999 on %hostname%" + $template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'" + $ActionMailSubject mailSubject + $ActionExecOnlyOnceEveryInterval 30 + :ommail:;RSYSLOG_SyslogProtocol23Format + } +</pre> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> + +</body></html> diff --git a/doc/mmfields.html b/doc/mmfields.html new file mode 100644 index 00000000..885d6bca --- /dev/null +++ b/doc/mmfields.html @@ -0,0 +1,91 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>Field Extraction Module (mmfields)</title></head> + +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>Fields Extraction Module (mmfields)</h1> +<p><b>Module Name: mmfields</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available since</b>: 7.5.1</p> +<p><b>Description</b>:</p> +<p>The mmfield module permits to extract fields. It is an alternate to using +the property replacer field extraction capabilities. In contrast to the +property replacer, all fields are extracted as once and stored inside the +structured data part (more precisely: they become Lumberjack [JSON] properties). +<p>Using this module is of special advantage if a field-based log format +is to be processed, like for example CEF <b>and</b> and either a large +number of fields is needed or a specific field is used multiple times +inside filters. In these scenarios, mmfields potentially offers better +performance than the property replacer of the RainerScript field extraction +method. The reason is that mmfields extracts all fields as one big sweep, +whereas the other methods extract fields individually, which requires +multiple passes through the same data. On the other hand, adding field +content to the rsyslog property dictionary also has some overhead, +so for high-performance use cases it is suggested to do some performance +testing before finally deciding which method to use. This is most important +if only a smaller subset of the fields is actually needed. +<p>In any case, mmfields provides a very handy and easy to use way to +parse structured data into a it's individual data items. Again, a primiary +use case was support for CEF (Common Event Format), which is made +extremely easy to do with this module. +<p>This module is implemented via the action interface. Thus it +can be conditionally used depending on some prequisites. +<p> </p> + +<p><b>Module Configuration Parameters</b>:</p> +<p>Currently none. +<p> </p> +<p><b>Action Confguration Parameters</b>:</p> +<ul> +<li><b>separator</b> - separatorChar (default ',')<br> +This is the character used to separate fields. Currently, only a single +character is permitted, while the RainerScript method permits to +specify multi-character separator strings. For CEF, this is not required. +If there is actual need to support multi-character separator strings, +support can relatively easy be added. It is suggested to request it on the +rsyslog mailing list, together with the use case - we intend to add +functionality only if there is a real use case behind the request +(in the past we too-often implemented things that actually never got used). +<br>The fields are named f<i>nbr</i>, where <i>nbr</i> is the field number +starting with one and being incremented for each field. +<li><b>jsonRoot</b> - path (default "!")<br> +This parameters specifies into which json path the extracted fields shall +be written. The default is to use the json root object itself. +</ul> + +<p><b>Caveats/Known Bugs:</b> +<ul> +<li>Currently none. +</ul> + +<p><b>Samples:</b></p> +<p>This is a very simple use case where each message is +parsed. The default separator character of comma is being used. +<p><textarea rows="5" cols="60">module(load="mmfields") +template(name="ftpl" type=string string="%$!%\n") +action(type="omfields") +action(type="omfile" file="/path/to/logfile" template="ftpl") +</textarea> + +<p>The following sample is similar to the previous one, but +this time the colon is used as separator and data is written +into the "$!mmfields" json path. +<p><textarea rows="5" cols="60">module(load="mmfields") +template(name="ftpl" type=string string="%$!%\n") +action(type="omfields" separator=":" jsonRoot="!mmfields") +action(type="omfile" file="/path/to/logfile" template="ftpl") +</textarea> + + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> + +</body></html> diff --git a/doc/mmnormalize.html b/doc/mmnormalize.html index 787bd957..911d6c89 100644 --- a/doc/mmnormalize.html +++ b/doc/mmnormalize.html @@ -17,7 +17,7 @@ a normal form. This is done so quickly, that it should be possible to normalize events in realtime. <p>This module is implemented via the output module interface. This means that mmnormalize should be called just like an action. After it has been called, -the normalized message properties are avaialable and can be accessed. These properties +the normalized message properties are available and can be accessed. These properties are called the "CEE/lumberjack" properties, because liblognorm creates a format that is inspired by the CEE/lumberjack approach. <p><b>Please note:</b> CEE/lumberjack properties are different from regular properties. diff --git a/doc/mmsnmptrapd.html b/doc/mmsnmptrapd.html index 699049d3..fb50f6c6 100644 --- a/doc/mmsnmptrapd.html +++ b/doc/mmsnmptrapd.html @@ -34,8 +34,8 @@ The following logic is applied to all message being processed: snmptrapd/severity/hostname. A configurable mapping table will be used to drive a new severity value from that severity string. If no mapping has been defined, the original severity is not changed. -<li>It replaces the "FromHost" value with the derived value from step2 -<li>It replaces the "Severity" value with the derived value from step 3 +<li>It replaces the "FromHost" value with the derived value from step 2 +<li>It replaces the "Severity" value with the derived value from step 3 </ol> <p>Note that the placement of this module inside the configuration is important. All actions before this modules is called will work on the unmodified message. All messages after it's call @@ -55,11 +55,11 @@ tells the module which start string inside the tag to look for. The default is matching incoming messages. It MUST not be given, except if two slashes are required for whatever reasons (so "tag/" results in a check for "tag//" at the start of the tag field). -<li><b>$mmsnmptrapdSeverityMapping</b> [severtiymap]<br> +<li><b>$mmsnmptrapdSeverityMapping</b> [severitymap]<br> This specifies the severity mapping table. It needs to be specified as a list. Note that due to the current config system <b>no whitespace</b> is supported inside the list, so be sure not to use any whitespace inside it.<br> -The list is constructed of Severtiy-Name/Severity-Value pairs, delimited by comma. +The list is constructed of Severity-Name/Severity-Value pairs, delimited by comma. Severity-Name is a case-sensitive string, e.g. "warning" and an associated numerical value (e.g. 4). Possible values are in the rage 0..7 and are defined in RFC5424, table 2. The diff --git a/doc/modules.html b/doc/modules.html index 4eae6db3..0ed7d4fe 100644 --- a/doc/modules.html +++ b/doc/modules.html @@ -70,7 +70,7 @@ other internal structures). Besides security, this also greatly simplifies the job of the output module developer.</p> <h2>Action Selectors</h2> <p>Modules (and rsyslog) need to know when they are called. For this, there must -a an action identification in selector lines. There are two syntaxes: the +be an action identification in selector lines. There are two syntaxes: the single-character syntax, where a single characters identifies a module (e.g. "*" for a wall message) and the modules designator syntax, where the module name is given between colons (e.g. ":ommysql:"). The single character syntax is diff --git a/doc/multi_ruleset.html b/doc/multi_ruleset.html index 83c495ca..14a761c5 100644 --- a/doc/multi_ruleset.html +++ b/doc/multi_ruleset.html @@ -5,7 +5,7 @@ <h1>Multiple Rulesets in rsyslog</h1> <p>Starting with version 4.5.0 and 5.1.1, <a href="http://www.rsyslog.com">rsyslog</a> supports multiple rulesets within a single configuration. -This is especially useful for routing the recpetion of remote messages to a set of specific rules. +This is especially useful for routing the reception of remote messages to a set of specific rules. Note that the input module must support binding to non-standard rulesets, so the functionality may not be available with all inputs. <p>In this document, I am using <a href="imtcp.html">imtcp</a>, an input module @@ -39,7 +39,7 @@ is the name space reserved for rsyslog use). If it finds this directive, it begi rule set (if the name was not yet know) or switches to an already-existing one (if the name was known). All rules defined between this $RuleSet directive and the next one are appended to the named ruleset. Note that the reserved name "RSYSLOG_DefaultRuleset" is used to -specify rsyslogd's default ruleset. You can use that name whereever you can use a ruleset name, +specify rsyslogd's default ruleset. You can use that name wherever you can use a ruleset name, including when binding an input to it. <p>Inside a ruleset, messages are processed as described above: they start with the first rule @@ -48,7 +48,7 @@ there are no more rules or the discard action is executed. Note that with multip no longer <b>all</b> rsyslog.conf rules are executed but <b>only</b> those that are contained within the specific ruleset. -<p>Inputs must explicitely bind to rulesets. If they don't do, the default ruleset is bound. +<p>Inputs must explicitly bind to rulesets. If they don't do, the default ruleset is bound. <p>This brings up the next question: @@ -58,10 +58,10 @@ it means that a specific input, or part of an input (like a tcp listener) will u ruleset to "pass its messages to". So when a new message arrives, it will be processed via the bound ruleset. Rule from all other rulesets are irrelevant and will never be processed. <p>This makes multiple rulesets very handy to process local and remote message via -seperate means: bind the respective receivers to different rule sets, and you do not need -to seperate the messages by any other method. +separate means: bind the respective receivers to different rule sets, and you do not need +to separate the messages by any other method. -<p>Binding to rulesets is input-specifc. For imtcp, this is done via the +<p>Binding to rulesets is input-specific. For imtcp, this is done via the <pre>input(type="imptcp" port="514" ruleset="rulesetname"); </pre> @@ -72,7 +72,7 @@ I personally think that it is best to define all rule sets at the top of rsyslog define the inputs at the bottom. This kind of reverses the traditional recommended ordering, but seems to be a really useful and straightforward way of doing things. <h2>Why are rulesets important for different parser configurations?</h2> -<p>Custom message parsers, used to handle differnet (and potentially otherwise-invalid) +<p>Custom message parsers, used to handle different (and potentially otherwise-invalid) message formats, can be bound to rulesets. So multiple rulesets can be a very useful way to handle devices sending messages in different malformed formats in a consistent way. Unfortunately, this is not uncommon in the syslog world. An in-depth explanation @@ -222,7 +222,7 @@ needs to insert messages into the main message queue. So if each of them wants t submit a newly arrived message into the queue at the same time, only one can do so while the others need to wait. With multiple rulesets, its own queue can be created for each ruleset. If now each listener is bound to its own ruleset, concurrent message submission is -possible. On a machine with a sufficiently large number of corse, this can result in +possible. On a machine with a sufficiently large number of cores, this can result in dramatic performance improvement. <p>It is highly advised that high-performance systems define a dedicated ruleset, with a dedicated queue for each of the inputs. diff --git a/doc/multi_ruleset_legacy_format.html b/doc/multi_ruleset_legacy_format.html index 273a4a09..03586ca7 100644 --- a/doc/multi_ruleset_legacy_format.html +++ b/doc/multi_ruleset_legacy_format.html @@ -5,18 +5,18 @@ <h1>Multiple Rulesets in rsyslog</h1> <p>Starting with version 4.5.0 and 5.1.1, <a href="http://www.rsyslog.com">rsyslog</a> supports multiple rulesets within a single configuration. -This is especially useful for routing the recpetion of remote messages to a set of specific rules. +This is especially useful for routing the reception of remote messages to a set of specific rules. Note that the input module must support binding to non-standard rulesets, so the functionality -may not be available with all inputs.<p> +may not be available with all inputs.</p> <b>Attention: this guide is shortened and only contains the samples in legacy format.</b> -Please follow this link to the full guide in the new config format "list": <a href="http://www.rsyslog.com/doc/multi_ruleset.html">http://www.rsyslog.com/doc/multi_ruleset.html<a> +Please follow this link to the full guide in the new config format "list": <a href="http://www.rsyslog.com/doc/multi_ruleset.html">http://www.rsyslog.com/doc/multi_ruleset.html</a> <h2>Examples</h2> <h3>Split local and remote logging</h3> <p>Let's say you have a pretty standard system that logs its local messages to the usual bunch of files that are specified in the default rsyslog.conf. As an example, your rsyslog.conf -might look like this: +might look like this:</p> <pre> # ... module loading ... @@ -34,7 +34,7 @@ cron.* /var/log/cron <p>Now, you want to add receive messages from a remote system and log these to a special file, but you do not want to have these messages written to the files specified above. The traditional approach is to add a rule in front of all others that -filters on the message, processes it and then discards it: +filters on the message, processes it and then discards it:</p> <pre> # ... module loading ... @@ -55,7 +55,7 @@ cron.* /var/log/cron </pre> <p>Note the tilde character, which is the discard action!. Also note that we assume that -192.0.2.1 is the sole remote sender (to keep it simple). +192.0.2.1 is the sole remote sender (to keep it simple).</p> <p>With multiple rulesets, we can simply define a dedicated ruleset for the remote reception case and bind it to the receiver. This may be written as follows: @@ -88,7 +88,7 @@ cron.* /var/log/cron <p>Here, we need to switch back to the default ruleset after we have defined our custom one. This is why I recommend a different ordering, which I find more intuitive. The sample -below has it, and it leads to the same results: +below has it, and it leads to the same results:</p> <pre> # ... module loading ... @@ -116,27 +116,27 @@ $InputTCPServerRun 10514 </pre> <p>Here, we do not switch back to the default ruleset, because this is not needed as it is -completely defined when we begin the "remote" ruleset. +completely defined when we begin the "remote" ruleset.</p> <p>Now look at the examples and compare them to the single-ruleset solution. You will notice that we do <b>not</b> need a real filter in the multi-ruleset case: we can simply use "*.*" as all messages now means all messages that are being processed by this rule set and all of them come in via the TCP receiver! This is what makes using multiple -rulesets so much easier. +rulesets so much easier.</p> <h3>Split local and remote logging for three different ports</h3> <p>This example is almost like the first one, but it extends it a little bit. While it is very similar, I hope it is different enough to provide a useful example why you may want -to have more than two rulesets. +to have more than two rulesets.</p> <p>Again, we would like to use the "regular" log files for local logging, only. But this time we set up three syslog/tcp listeners, each one listening to a different port (in this example 10514, 10515, and 10516). Logs received from these receivers shall go into different files. Also, logs received from 10516 (and only from that port!) with "mail.*" priority, shall be written into a specif file and <b>not</b> be -written to 10516's general log file. +written to 10516's general log file.</p> -<p>This is the config: +<p>This is the config:</p> <pre> # ... module loading ... @@ -180,12 +180,12 @@ $InputTCPServerRun 10516 </pre> <p>Note that the "mail.*" rule inside the "remote10516" ruleset does -not affect processing inside any other rule set, including the default rule set. +not affect processing inside any other rule set, including the default rule set.</p> <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> -project.<br> +project.<br/> Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/ns_gtls.html b/doc/ns_gtls.html index 0d02ad02..21a7f19c 100644 --- a/doc/ns_gtls.html +++ b/doc/ns_gtls.html @@ -10,7 +10,7 @@ library</a>.</p> <p><b>Available since:</b> 3.19.0 (suggested minimum 3.19.8 and above)</p> <p style="font-weight: bold;">Supported Driver Modes</p> <ul> -<li>0 - unencrypted trasmission (just like <a href="ns_ptcp.html">ptcp</a> driver)</li> +<li>0 - unencrypted transmission (just like <a href="ns_ptcp.html">ptcp</a> driver)</li> <li>1 - TLS-protected operation</li> </ul> Note: mode 0 does not provide any benefit over the ptcp driver. This @@ -38,7 +38,7 @@ unauthorized access. It is recommended NOT to use this mode.</p> <p>x509/certvalid is a nonstandard mode. It validates the remote peers certificate, but does not check the subject name. This is weak authentication that may be useful in scenarios where multiple -devices are deployed and it is sufficient proof of authenticy when +devices are deployed and it is sufficient proof of authenticity when their certificates are signed by the CA the server trusts. This is better than anon authentication, but still not recommended. <b>Known Problems</b><br> diff --git a/doc/omfwd.html b/doc/omfwd.html index 53f9e527..51aa58b5 100644 --- a/doc/omfwd.html +++ b/doc/omfwd.html @@ -35,7 +35,38 @@ Framing-Mode to be for forwarding. This affects only TCP-based protocols. It is ignored for UDP. In protocol engineering, ``framing'' means how multiple messages over the same connection are separated. Usually, this is transparent to users. Unfortunately, the early syslog protocol evolved, and so there are cases where users need to specify the framing. The traditional framing is nontransparent. With it, messages are end when a LF (aka ``line break'', ``return'') is encountered, and the next message starts immediately after the LF. If multi-line messages are received, these are essentially broken up into multiple message, usually with all but the first message segment being incorrectly formatted. The octet-counting framing solves this issue. With it, each message is prefixed with the actual message length, so that a receivers knows exactly where the message ends. Multi-line messages cause no problem here. This mode is very close to the method described in RFC5425 for TLS-enabled syslog. Unfortunately, only few syslogd implementations support octet-counted framing. As such, the traditional framing is set as default, even though it has defects. If it is known that the receiver supports octet-counted framing, it is suggested to use that framing mode. <br></li><br> <li><strong>ZipLevel </strong>0..9 [default 0]<br> - Compression level for messages. Rsyslog implements a proprietary capability to zip transmitted messages. Note that compression happens on a message-per-message basis. As such, there is a performance gain only for larger messages. Before compressing a message, rsyslog checks if there is some gain by compression. If so, the message is sent compressed. If not, it is sent uncompressed. As such, it is totally valid that compressed and uncompressed messages are intermixed within a conversation. <br>The compression level is specified via the usual factor of 0 to 9, with 9 being the strongest compression (taking up most processing time) and 0 being no compression at all (taking up no extra processing time). <br></li><br> + Compression level for messages. + <br>Up until rsyslog 7.5.1, this was the only compression setting that + rsyslog understood. Starting with 7.5.1, we have different compression + modes. All of them are affected by the ziplevel. If, however, no mode + is explicitely set, setting ziplevel also turns on "single" + compression mode, so pre 7.5.1 configuration will continue to work + as expected. + <br>The compression level is specified via the usual factor of 0 to 9, with 9 being the strongest compression (taking up most processing time) and 0 being no compression at all (taking up no extra processing time). <br></li><br> + <li><b>compression.mode</b><i>mode</i><br> + <i>mode</i> is one of "none", "single", or "stream:always". The + default is "none", in which no compression happens at all. + <br>In "single" compression mode, Rsyslog implements a proprietary + capability to zip transmitted messages. That compression happens + on a message-per-message basis. As such, there is a performance gain + only for larger messages. Before compressing a message, rsyslog checks + if there is some gain by compression. If so, the message is sent + compressed. If not, it is sent uncompressed. As such, it is totally + valid that compressed and uncompressed messages are intermixed + within a conversation. + <br>In "stream:always" compression mode the full stream is being + compressed. This also uses non-standard protocol and is compatible + only with receives that have the same abilities. This mode offers + potentially very high compression ratios. With typical syslog + messages, it can be as high as 95+% compression (so only one twentieth + of data is actually transmitted!). Note that this mode introduces + extra latency, as data is only sent when the compressor emits new + compressed data. For typical syslog messages, this can mean that + some hundered messages may be held in local buffers before they are + actually sent. This mode has been introduced in 7.5.1. + <br><b>Note: currently only imptcp supports receiving stream-compressed + data.</b> + <br></li><br> <li><strong>RebindInterval </strong>integer<br> Permits to specify an interval at which the current connection is broken and re-established. This setting is primarily an aid to load balancers. After the configured number of messages has been transmitted, the current connection is terminated and a new one started. Note that this setting applies to both TCP and UDP traffic. For UDP, the new ``connection'' uses a different source port (ports are cycled and not reused too frequently). This usually is perceived as a ``new connection'' by load balancers, which in turn forward messages to another physical target system. <br></li><br> diff --git a/doc/omprog.html b/doc/omprog.html index 471ab224..4f369735 100644 --- a/doc/omprog.html +++ b/doc/omprog.html @@ -25,7 +25,34 @@ con re-using existing binaries. For the time being, it simply is not done. In th we may add an option for such pooling, provided that some demand for that is voiced. You can also mimic the same effect by defining multiple rulesets and including them (at the price of some slight performance loss). -<p><b>Configuration Directives</b>:</p> + +<p> </p> + +<p><b>Module Parameters</b>:</p> +<ul> + <li><strong>Template </strong>[templateName]<br> + sets a new default template for file actions.<br></li> + +</ul> +<p> </p> +<p><b>Action Parameters</b>:</p> +<ul> + <li><strong>binary </strong><br> + Mostly equivalent to the "binary" action parameter, but must contain the binary name + only. In legacy config, it is <b>not possible</b> to specify command line parameters. +</ul> +<p><b>Caveats/Known Bugs:</b></p><ul><li>None.</li></ul> +<p><b>Sample:</b></p> +<p>The following command writes all syslog messages into a file.</p> +<textarea rows="5" cols="85">Module (load="omprog") +*.* action(type="omprog" + binary="/pathto/omprog.py --parm1=\"value 1\" --parm2=value2" + template="RSYSLOG_TraditionalFileFormat") +</textarea> + +<br><br> + +<p><b>Legacy Configuration Directives</b>:</p> <ul> <li><b>$ActionOMProgBinary</b> <binary><br> The binary program to be executed. @@ -36,7 +63,7 @@ The binary program to be executed. [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/omrelp.html b/doc/omrelp.html index 8858f884..8049ebaf 100644 --- a/doc/omrelp.html +++ b/doc/omrelp.html @@ -24,16 +24,89 @@ implementation).</p> rsyslog 7.3.10. For older versions, legacy configuration directives must be used. <ul> - <li><b>target </b>(mandatory)<br> + <li><b>target</b> (mandatory)<br> The target server to connect to. </li> - <li><b>template </b>(not mandatory, default "RSYSLOG_ForwardFormat")<br> + <li><b>template</b> (not mandatory, default "RSYSLOG_ForwardFormat")<br> Defines the template to be used for the output. </li> - <li><b>timeout </b>(not mandatory, default 90)<br> + <li><b>timeout</b> (not mandatory, default 90)<br> Timeout for relp sessions. If set too low, valid sessions may be considered dead and tried to recover. </li> + <li><b>windowSize</b> (not mandatory, default 0)<br> + This is an <b>expert parameter</b>. It permits to override the + RELP window size being used by the client. Changing the window + size has both an effect on performance as well as potential + message duplication in failure case. A larger window size means + more performance, but also potentially more duplicated + messages - and vice versa. The default 0 means that librelp's + default window size is being used, which is considered a + compromise between goals reached. For your information: + at the time of this writing, the librelp default window size + is 128 messages, but this may change at any time. + <br>Note that there is no equivalent server parameter, as the + client proposes and manages the window size in RELP protocol. + <li><b>tls</b> (not mandatory, values "on","off", default "off")<br> + If set to "on", the RELP connection will be encrypted by TLS, so that the data is protected against observers. Please note that both the client and the server must have set TLS to either "on" or "off". Other combinations lead to unpredictable results. + </li> + <li><b>tls.compression</b> (not mandatory, values "on","off", default "off")<br> + The controls if the TLS stream should be compressed (zipped). While this + increases CPU use, the network bandwidth should be reduced. Note that + typical text-based log records usually compress rather well. + </li> + <li><b>tls.permittedPeer</b> peer</br> + Places access restrictions on this forwarder. Only peers which + have been listed in this parameter may be connected to. + This guards against rouge servers and man-in-the-middle + attacks. The validation + bases on the certficate the remote peer presents.<br> + The <i>peer</i> parameter lists permitted certificate + fingerprints. Note that it is an array parameter, so either + a single or multiple fingerprints can be listed. When a + non-permitted peer is connected to, the refusal is logged together + with it's fingerprint. So if the administrator knows this was + a valid request, he can simple add the fingerprint by copy and + paste from the logfile to rsyslog.conf. It must be noted, though, + that this situation should usually not happen after initial + client setup and administrators should be alert in this case. + <br>Note that usually a single remote peer should be all that + is ever needed. Support for multiple peers is primarily included + in support of load balancing scenarios. If the connection + goes to a specific server, only one specific certificate is ever + expected (just like when connecting to a specific ssh server). + <br>To specify multiple fingerprints, just enclose them + in braces like this: + <br>tls.permittedPeer=["SHA1:...1", "SHA1:....2"] + <br>To specify just a single peer, you can either + specify the string directly or enclose it in braces. + </li> + <li><b>tls.authMode</b> mode</br> + Sets the mode used for mutual authentication. Supported values are + either "<i>fingerprint</i>" or "<i>name"</i>. + <br>Fingerprint mode basically is what SSH + does. It does not require a full PKI to be present, instead self-signed + certs can be used on all peers. Even if a CA certificate is given, the + validity of the peer cert is NOT verified against it. Only the + certificate fingerprint counts. + <br>In "name" mode, certificate validation happens. Here, the matching + is done against the certificate's subjectAltName and, as a fallback, + the subject common name. If the certificate contains multiple names, + a match on any one of these names is considered good and permits the + peer to talk to rsyslog. + <li><b>tls.prioritystring</b> (not mandatory, string)<br> + This parameter permits to specify the so-called "priority string" to + GnuTLS. This string gives complete control over all crypto parameters, + including compression setting. For this reason, when the prioritystring + is specified, the "tls.compression" parameter has no effect and is + ignored. + <br>Full information about how to construct a priority string can be + found in the GnuTLS manual. At the time of this writing, this + information was contained in + <a href="http://gnutls.org/manual/html_node/Priority-Strings.html">section 6.10 of the GnuTLS manual</a>. + <br><b>Note: this is an expert parameter.</b> Do not use if you do + not exactly know what you are doing. + </li> </ul> <p><b>Sample:</b></p> <p>The following sample sends all messages to the central server diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 18d6b8a1..b6b0748b 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -111,6 +111,9 @@ probably an excellent starting base for writing a new module. Currently, the fol modules exist inside the source tree: <ul> <li><a href="mmanon.html">mmanon</a> - used to anonymize log messages. +<li><a href="mmcount.html">mmcount</a> - message modification plugin which counts messages +<li><a href="mmfields.html">mmfields</a> - used to extract fields from +specially formatted messages (e.g. CEF) <li><a href="mmnormalize.html">mmnormalize</a> - used to normalize log messages. Note that this actually is a <b>generic</b> module. <li><a href="mmjsonparse.html">mmjsonparse</a> - used to interpret CEE/lumberjack diff --git a/grammar/.gitignore b/grammar/.gitignore new file mode 100644 index 00000000..8bd546bd --- /dev/null +++ b/grammar/.gitignore @@ -0,0 +1 @@ +lexer.c diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 906521dd..d3a29470 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -10,7 +10,7 @@ * * File begun on 2010-08-10 by RGerhards * - * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -50,6 +50,8 @@ #include <sys/socket.h> #include <sys/epoll.h> #include <netinet/tcp.h> +#include <stdint.h> +#include <zlib.h> #if HAVE_FCNTL_H #include <fcntl.h> #endif @@ -93,6 +95,11 @@ static void * wrkr(void *myself); #define DFLT_wrkrMax 2 +#define COMPRESS_NEVER 0 +#define COMPRESS_SINGLE_MSG 1 /* old, single-message compression */ +/* all other settings are for stream-compression */ +#define COMPRESS_STREAM_ALWAYS 2 + /* config settings */ typedef struct configSettings_s { int bKeepAlive; /* support keep-alive packets */ @@ -117,6 +124,7 @@ struct instanceConf_s { int bEmitMsgOnClose; int bSuppOctetFram; /* support octet-counted framing? */ int iAddtlFrameDelim; + uint8_t compressionMode; uchar *pszBindPort; /* port to bind to */ uchar *pszBindAddr; /* IP to bind socket to */ uchar *pszBindRuleset; /* name of ruleset to bind to */ @@ -156,6 +164,7 @@ static struct cnfparamdescr inppdescr[] = { { "ruleset", eCmdHdlrString, 0 }, { "supportoctetcountedframing", eCmdHdlrBinary, 0 }, { "notifyonconnectionclose", eCmdHdlrBinary, 0 }, + { "compression.mode", eCmdHdlrGetWord, 0 }, { "keepalive", eCmdHdlrBinary, 0 }, { "keepalive.probes", eCmdHdlrInt, 0 }, { "keepalive.time", eCmdHdlrInt, 0 }, @@ -191,6 +200,7 @@ struct ptcpsrv_s { int iKeepAliveIntvl; int iKeepAliveProbes; int iKeepAliveTime; + uint8_t compressionMode; uchar *pszInputName; prop_t *pInputName; /* InputName in (fast to process) property format */ ruleset_t *pRuleset; @@ -207,11 +217,13 @@ struct ptcpsrv_s { * includes support for doubly-linked list. */ struct ptcpsess_s { -// ptcpsrv_t *pSrv; /* our server TODO: check remove! */ ptcplstn_t *pLstn; /* our listener */ ptcpsess_t *prev, *next; int sock; epolld_t *epd; + sbool bzInitDone; /* did we do an init of zstrm already? */ + z_stream zstrm; /* zip stream to use for tcp compression */ + uint8_t compressionMode; //--- from tcps_sess.h int iMsg; /* index of next char to store in msg */ int bAtStrtOfFram; /* are we at the very beginning of a new frame? */ @@ -239,6 +251,8 @@ struct ptcplstn_s { sbool bSuppOctetFram; epolld_t *epd; statsobj_t *stats; /* listener stats */ + intctr_t rcvdBytes; + intctr_t rcvdDecompressed; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) }; @@ -806,19 +820,19 @@ processDataRcvd(ptcpsess_t *pThis, char c, struct syslogTime *stTime, time_t ttG * EXTRACT from tcps_sess.c */ static rsRetVal -DataRcvd(ptcpsess_t *pThis, char *pData, size_t iLen) +DataRcvdUncompressed(ptcpsess_t *pThis, char *pData, size_t iLen, time_t ttGenTime) { multi_submit_t multiSub; msg_t *pMsgs[CONF_NUM_MULTISUB]; struct syslogTime stTime; - time_t ttGenTime; char *pEnd; DEFiRet; assert(pData != NULL); assert(iLen > 0); - datetime.getCurrTime(&stTime, &ttGenTime); + if(ttGenTime == 0) + datetime.getCurrTime(&stTime, &ttGenTime); multiSub.ppMsgs = pMsgs; multiSub.maxElem = CONF_NUM_MULTISUB; multiSub.nElem = 0; @@ -836,6 +850,71 @@ finalize_it: RETiRet; } +static rsRetVal +DataRcvdCompressed(ptcpsess_t *pThis, char *buf, size_t len) +{ + struct syslogTime stTime; + time_t ttGenTime; + int zRet; /* zlib return state */ + unsigned outavail; + uchar zipBuf[64*1024]; // TODO: alloc on heap, and much larger (512KiB? batch size!) + DEFiRet; + // TODO: can we do stats counters? Even if they are not 100% correct under all cases, + // by simply updating the input and output sizes? + uint64_t outtotal; + + assert(iLen > 0); + + datetime.getCurrTime(&stTime, &ttGenTime); + outtotal = 0; + + if(!pThis->bzInitDone) { + /* allocate deflate state */ + pThis->zstrm.zalloc = Z_NULL; + pThis->zstrm.zfree = Z_NULL; + pThis->zstrm.opaque = Z_NULL; + zRet = inflateInit(&pThis->zstrm); + if(zRet != Z_OK) { + DBGPRINTF("imptcp: error %d returned from zlib/inflateInit()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + pThis->bzInitDone = RSTRUE; + } + + pThis->zstrm.next_in = (Bytef*) buf; + pThis->zstrm.avail_in = len; + /* run inflate() on buffer until everything has been uncompressed */ + do { + DBGPRINTF("imptcp: in inflate() loop, avail_in %d, total_in %ld\n", pThis->zstrm.avail_in, pThis->zstrm.total_in); + pThis->zstrm.avail_out = sizeof(zipBuf); + pThis->zstrm.next_out = zipBuf; + zRet = inflate(&pThis->zstrm, Z_NO_FLUSH); /* no bad return value */ + DBGPRINTF("after inflate, ret %d, avail_out %d\n", zRet, pThis->zstrm.avail_out); + outavail = sizeof(zipBuf) - pThis->zstrm.avail_out; + if(outavail != 0) { + outtotal += outavail; + pThis->pLstn->rcvdDecompressed += outavail; + CHKiRet(DataRcvdUncompressed(pThis, (char*)zipBuf, outavail, ttGenTime)); + } + } while (pThis->zstrm.avail_out == 0); + + dbgprintf("end of DataRcvCompress, sizes: in %lld, out %llu\n", (long long) len, outtotal); +finalize_it: + RETiRet; +} + +static rsRetVal +DataRcvd(ptcpsess_t *pThis, char *pData, size_t iLen) +{ + DEFiRet; + pThis->pLstn->rcvdBytes += iLen; + if(pThis->compressionMode >= COMPRESS_STREAM_ALWAYS) + iRet = DataRcvdCompressed(pThis, pData, iLen); + else + iRet = DataRcvdUncompressed(pThis, pData, iLen, 0); + RETiRet; +} + /****************************************** --END-- TCP SUPPORT FUNCTIONS ***********************************/ @@ -936,6 +1015,14 @@ addLstn(ptcpsrv_t *pSrv, int sock, int isIPv6) STATSCOUNTER_INIT(pLstn->ctrSubmit, pLstn->mutCtrSubmit); CHKiRet(statsobj.AddCounter(pLstn->stats, UCHAR_CONSTANT("submitted"), ctrType_IntCtr, &(pLstn->ctrSubmit))); + /* the following counters are not protected by mutexes; we accept + * that they may not be 100% correct */ + pLstn->rcvdBytes = 0, + pLstn->rcvdDecompressed = 0; + CHKiRet(statsobj.AddCounter(pLstn->stats, UCHAR_CONSTANT("bytes.received"), + ctrType_IntCtr, &(pLstn->rcvdBytes))); + CHKiRet(statsobj.AddCounter(pLstn->stats, UCHAR_CONSTANT("bytes.decompressed"), + ctrType_IntCtr, &(pLstn->rcvdDecompressed))); CHKiRet(statsobj.ConstructFinalize(pLstn->stats)); /* add to start of server's listener list */ @@ -948,6 +1035,7 @@ addLstn(ptcpsrv_t *pSrv, int sock, int isIPv6) iRet = addEPollSock(epolld_lstn, pLstn, sock, &pLstn->epd); finalize_it: +dbgprintf("DDDD: addLstn return %d\n", iRet); RETiRet; } @@ -968,9 +1056,11 @@ addSess(ptcplstn_t *pLstn, int sock, prop_t *peerName, prop_t *peerIP) pSess->bSuppOctetFram = pLstn->bSuppOctetFram; pSess->inputState = eAtStrtFram; pSess->iMsg = 0; + pSess->bzInitDone = 0; pSess->bAtStrtOfFram = 1; pSess->peerName = peerName; pSess->peerIP = peerIP; + pSess->compressionMode = pLstn->pSrv->compressionMode; /* add to start of server's listener list */ pSess->prev = NULL; @@ -988,6 +1078,44 @@ finalize_it: } +/* finish zlib buffer, to be called before closing the session. + */ +static rsRetVal +doZipFinish(ptcpsess_t *pSess) +{ + int zRet; /* zlib return state */ + DEFiRet; + unsigned outavail; + uchar zipBuf[32*1024]; // TODO: use "global" one from pSess + + if(!pSess->bzInitDone) + goto done; + + pSess->zstrm.avail_in = 0; + /* run inflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("doZipFinish: in inflate() loop, avail_in %d, total_in %ld\n", pSess->zstrm.avail_in, pSess->zstrm.total_in); + pSess->zstrm.avail_out = sizeof(zipBuf); + pSess->zstrm.next_out = zipBuf; + zRet = inflate(&pSess->zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after inflate, ret %d, avail_out %d\n", zRet, pSess->zstrm.avail_out); + outavail = sizeof(zipBuf) - pSess->zstrm.avail_out; + if(outavail != 0) { + pSess->pLstn->rcvdDecompressed += outavail; + CHKiRet(DataRcvdUncompressed(pSess, (char*)zipBuf, outavail, 0)); // TODO: query time! + } + } while (pSess->zstrm.avail_out == 0); + +finalize_it: + zRet = inflateEnd(&pSess->zstrm); + if(zRet != Z_OK) { + DBGPRINTF("imptcp: error %d returned from zlib/inflateEnd()\n", zRet); + } + + pSess->bzInitDone = 0; +done: RETiRet; +} + /* close/remove a session * NOTE: we must first remove the fd from the epoll set and then close it -- else we * get an error "bad file descriptor" from epoll. @@ -998,6 +1126,9 @@ closeSess(ptcpsess_t *pSess) int sock; DEFiRet; + if(pSess->compressionMode >= COMPRESS_STREAM_ALWAYS) + doZipFinish(pSess); + sock = pSess->sock; CHKiRet(removeEPollSock(sock, pSess->epd)); close(sock); @@ -1048,6 +1179,7 @@ createInstance(instanceConf_t **pinst) inst->pBindRuleset = NULL; inst->ratelimitBurst = 10000; /* arbitrary high limit */ inst->ratelimitInterval = 0; /* off */ + inst->compressionMode = COMPRESS_SINGLE_MSG; /* node created, let's add to config */ if(loadModConf->tail == NULL) { @@ -1127,6 +1259,7 @@ addListner(modConfData_t __attribute__((unused)) *modConf, instanceConf_t *inst) pSrv->iKeepAliveProbes = inst->iKeepAliveProbes; pSrv->iKeepAliveTime = inst->iKeepAliveTime; pSrv->bEmitMsgOnClose = inst->bEmitMsgOnClose; + pSrv->compressionMode = inst->compressionMode; CHKiRet(ratelimitNew(&pSrv->ratelimiter, "imtcp", (char*)inst->pszBindPort)); ratelimitSetLinuxLike(pSrv->ratelimiter, inst->ratelimitInterval, inst->ratelimitBurst); ratelimitSetThreadSafe(pSrv->ratelimiter); @@ -1269,6 +1402,10 @@ sessActivity(ptcpsess_t *pSess) { int lenRcv; int lenBuf; + uchar *peerName; + int lenPeer; + int remsock = 0; /* init just to keep compiler happy... :-( */ + sbool bEmitOnClose = 0; char rcvBuf[128*1024]; DEFiRet; @@ -1285,13 +1422,15 @@ sessActivity(ptcpsess_t *pSess) } else if (lenRcv == 0) { /* session was closed, do clean-up */ if(pSess->pLstn->pSrv->bEmitMsgOnClose) { - uchar *peerName; - int lenPeer; - prop.GetString(pSess->peerName, &peerName, &lenPeer); - errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "imptcp session %d closed by remote peer %s.\n", - pSess->sock, peerName); + prop.GetString(pSess->peerName, &peerName, &lenPeer), + remsock = pSess->sock; + bEmitOnClose = 1; + } + CHKiRet(closeSess(pSess)); /* close may emit more messages in strmzip mode! */ + if(bEmitOnClose) { + errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "imptcp session %d closed by " + "remote peer %s.\n", remsock, peerName); } - CHKiRet(closeSess(pSess)); break; } else { if(errno == EAGAIN || errno == EWOULDBLOCK) @@ -1415,6 +1554,7 @@ wrkr(void *myself) BEGINnewInpInst struct cnfparamvals *pvals; instanceConf_t *inst; + char *cstr; int i; CODESTARTnewInpInst DBGPRINTF("newInpInst (imptcp)\n"); @@ -1443,6 +1583,19 @@ CODESTARTnewInpInst inst->pszBindRuleset = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(inppblk.descr[i].name, "supportoctetcountedframing")) { inst->bSuppOctetFram = (int) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "compression.mode")) { + cstr = es_str2cstr(pvals[i].val.d.estr, NULL); + if(!strcasecmp(cstr, "stream:always")) { + inst->compressionMode = COMPRESS_STREAM_ALWAYS; + } else if(!strcasecmp(cstr, "none")) { + inst->compressionMode = COMPRESS_NEVER; + } else { + errmsg.LogError(0, RS_RET_PARAM_ERROR, "omfwd: invalid value for 'compression.mode' " + "parameter (given is '%s')", cstr); + free(cstr); + ABORT_FINALIZE(RS_RET_PARAM_ERROR); + } + free(cstr); } else if(!strcmp(inppblk.descr[i].name, "keepalive")) { inst->bKeepAlive = (int) pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "keepalive.probes")) { @@ -1650,6 +1803,7 @@ shutdownSrv(ptcpsrv_t *pSrv) ptcplstn_t *pLstn, *lstnDel; ptcpsess_t *pSess, *sessDel; +dbgprintf("DDDD: enter shutdownSrv\n"); /* listeners */ pLstn = pSrv->pLstn; while(pLstn != NULL) { @@ -1658,7 +1812,9 @@ shutdownSrv(ptcpsrv_t *pSrv) /* now unlink listner */ lstnDel = pLstn; pLstn = pLstn->next; - DBGPRINTF("imptcp shutdown listen socket %d\n", lstnDel->sock); + DBGPRINTF("imptcp shutdown listen socket %d (rcvd %lld bytes, " + "decompressed %lld)\n", lstnDel->sock, lstnDel->rcvdBytes, + lstnDel->rcvdDecompressed); free(lstnDel->epd); free(lstnDel); } diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 5e0ae552..d04e41e1 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -4,7 +4,7 @@ * * File begun on 2008-03-13 by RGerhards * - * Copyright 2008-2012 Adiscon GmbH. + * Copyright 2008-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -47,6 +47,7 @@ #include "prop.h" #include "ruleset.h" #include "glbl.h" +#include "statsobj.h" MODULE_TYPE_INPUT MODULE_TYPE_NOKEEP @@ -59,6 +60,7 @@ DEFobjCurrIf(prop) DEFobjCurrIf(errmsg) DEFobjCurrIf(ruleset) DEFobjCurrIf(glbl) +DEFobjCurrIf(statsobj) /* forward definitions */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal); @@ -74,7 +76,31 @@ static struct configSettings_s { struct instanceConf_s { uchar *pszBindPort; /* port to bind to */ + sbool bEnableTLS; + sbool bEnableTLSZip; + int dhBits; + uchar *pristring; /* GnuTLS priority string (NULL if not to be provided) */ + uchar *authmode; /* TLS auth mode */ + uchar *caCertFile; + uchar *myCertFile; + uchar *myPrivKeyFile; + struct { + int nmemb; + uchar **name; + } permittedPeers; + struct instanceConf_s *next; + /* with librelp, this module does not have any own specific session + * or listener active data item. As a "work-around", we keep some + * data items inside the configuration object. To keep things + * decently clean, we put them all into their dedicated struct. So + * it is easy to judge what is actual configuration and what is + * dynamic runtime data. -- rgerhards, 2013-06-18 + */ + struct { + statsobj_t *stats; /* listener stats */ + STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) + } data; }; @@ -88,9 +114,28 @@ struct modConfData_s { static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current load process */ +/* module-global parameters */ +static struct cnfparamdescr modpdescr[] = { + { "ruleset", eCmdHdlrGetWord, 0 }, +}; +static struct cnfparamblk modpblk = + { CNFPARAMBLK_VERSION, + sizeof(modpdescr)/sizeof(struct cnfparamdescr), + modpdescr + }; + /* input instance parameters */ static struct cnfparamdescr inppdescr[] = { - { "port", eCmdHdlrString, CNFPARAM_REQUIRED } + { "port", eCmdHdlrString, CNFPARAM_REQUIRED }, + { "tls", eCmdHdlrBinary, 0 }, + { "tls.permittedpeer", eCmdHdlrArray, 0 }, + { "tls.authmode", eCmdHdlrString, 0 }, + { "tls.dhbits", eCmdHdlrInt, 0 }, + { "tls.prioritystring", eCmdHdlrString, 0 }, + { "tls.cacert", eCmdHdlrString, 0 }, + { "tls.mycert", eCmdHdlrString, 0 }, + { "tls.myprivkey", eCmdHdlrString, 0 }, + { "tls.compression", eCmdHdlrBinary, 0 } }; static struct cnfparamblk inppblk = { CNFPARAMBLK_VERSION, @@ -102,6 +147,30 @@ static struct cnfparamblk inppblk = /* ------------------------------ callbacks ------------------------------ */ +static void +onErr(void *pUsr, char *objinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) +{ + instanceConf_t *inst = (instanceConf_t*) pUsr; + errmsg.LogError(0, RS_RET_RELP_AUTH_FAIL, "imrelp[%s]: error '%s', object " + " '%s' - input may not work as intended", + inst->pszBindPort, errmesg, objinfo); +} + +static void +onGenericErr(char *objinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) +{ + errmsg.LogError(0, RS_RET_RELP_ERR, "imrelp: librelp error '%s', object " + " '%s' - input may not work as intended", errmesg, objinfo); +} + +static void +onAuthErr(void *pUsr, char *authinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) +{ + instanceConf_t *inst = (instanceConf_t*) pUsr; + errmsg.LogError(0, RS_RET_RELP_AUTH_FAIL, "imrelp[%s]: authentication error '%s', peer " + "is '%s'", inst->pszBindPort, errmesg, authinfo); +} + /* callback for receiving syslog messages. This function is invoked from the * RELP engine when a syslog message arrived. It must return a relpRetVal, * with anything else but RELP_RET_OK terminating the relp session. Please note @@ -113,10 +182,11 @@ static struct cnfparamblk inppblk = * we will only see the hostname (twice). -- rgerhards, 2009-10-14 */ static relpRetVal -onSyslogRcv(uchar *pHostname, uchar *pIP, uchar *msg, size_t lenMsg) +onSyslogRcv(void *pUsr, uchar *pHostname, uchar *pIP, uchar *msg, size_t lenMsg) { prop_t *pProp = NULL; msg_t *pMsg; + instanceConf_t *inst = (instanceConf_t*) pUsr; DEFiRet; CHKiRet(msgConstruct(&pMsg)); @@ -134,6 +204,7 @@ onSyslogRcv(uchar *pHostname, uchar *pIP, uchar *msg, size_t lenMsg) CHKiRet(MsgSetRcvFromIPStr(pMsg, pIP, ustrlen(pIP), &pProp)); CHKiRet(prop.Destruct(&pProp)); CHKiRet(submitMsg2(pMsg)); + STATSCOUNTER_INC(inst->data.ctrSubmit, inst->data.mutCtrSubmit); finalize_it: @@ -155,6 +226,15 @@ createInstance(instanceConf_t **pinst) inst->next = NULL; inst->pszBindPort = NULL; + inst->bEnableTLS = 0; + inst->bEnableTLSZip = 0; + inst->dhBits = 0; + inst->pristring = NULL; + inst->authmode = NULL; + inst->permittedPeers.nmemb = 0; + inst->caCertFile = NULL; + inst->myCertFile = NULL; + inst->myPrivKeyFile = NULL; /* node created, let's add to config */ if(loadModConf->tail == NULL) { @@ -179,7 +259,7 @@ std_checkRuleset_genErrMsg(modConfData_t *modConf, __attribute__((unused)) insta } -/* This function is called when a new listener instace shall be added to +/* This function is called when a new listener instance shall be added to * the current config object via the legacy config system. It just shuffles * all parameters to the listener in-memory instance. * rgerhards, 2011-05-04 @@ -204,19 +284,63 @@ finalize_it: static rsRetVal addListner(modConfData_t __attribute__((unused)) *modConf, instanceConf_t *inst) { + relpSrv_t *pSrv; + uchar statname[64]; + int i; DEFiRet; if(pRelpEngine == NULL) { CHKiRet(relpEngineConstruct(&pRelpEngine)); CHKiRet(relpEngineSetDbgprint(pRelpEngine, dbgprintf)); CHKiRet(relpEngineSetFamily(pRelpEngine, glbl.GetDefPFFamily())); CHKiRet(relpEngineSetEnableCmd(pRelpEngine, (uchar*) "syslog", eRelpCmdState_Required)); - CHKiRet(relpEngineSetSyslogRcv(pRelpEngine, onSyslogRcv)); + CHKiRet(relpEngineSetSyslogRcv2(pRelpEngine, onSyslogRcv)); + CHKiRet(relpEngineSetOnErr(pRelpEngine, onErr)); + CHKiRet(relpEngineSetOnGenericErr(pRelpEngine, onGenericErr)); + CHKiRet(relpEngineSetOnAuthErr(pRelpEngine, onAuthErr)); if (!glbl.GetDisableDNS()) { CHKiRet(relpEngineSetDnsLookupMode(pRelpEngine, 1)); } } - CHKiRet(relpEngineAddListner(pRelpEngine, inst->pszBindPort)); + CHKiRet(relpEngineListnerConstruct(pRelpEngine, &pSrv)); + CHKiRet(relpSrvSetLstnPort(pSrv, inst->pszBindPort)); + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&(inst->data.stats))); + snprintf((char*)statname, sizeof(statname), "imrelp(%s)", + inst->pszBindPort); + statname[sizeof(statname)-1] = '\0'; /* just to be on the save side... */ + CHKiRet(statsobj.SetName(inst->data.stats, statname)); + STATSCOUNTER_INIT(inst->data.ctrSubmit, inst->data.mutCtrSubmit); + CHKiRet(statsobj.AddCounter(inst->data.stats, UCHAR_CONSTANT("submitted"), + ctrType_IntCtr, &(inst->data.ctrSubmit))); + CHKiRet(statsobj.ConstructFinalize(inst->data.stats)); + /* end stats counters */ + relpSrvSetUsrPtr(pSrv, inst); + if(inst->bEnableTLS) { + relpSrvEnableTLS(pSrv); + if(inst->bEnableTLSZip) { + relpSrvEnableTLSZip(pSrv); + } + if(inst->dhBits) { + relpSrvSetDHBits(pSrv, inst->dhBits); + } + relpSrvSetGnuTLSPriString(pSrv, (char*)inst->pristring); + if(relpSrvSetAuthMode(pSrv, (char*)inst->authmode) != RELP_RET_OK) { + errmsg.LogError(0, RS_RET_RELP_ERR, + "imrelp: invalid auth mode '%s'\n", inst->authmode); + ABORT_FINALIZE(RS_RET_RELP_ERR); + } + if(relpSrvSetCACert(pSrv, (char*) inst->caCertFile) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(relpSrvSetOwnCert(pSrv, (char*) inst->myCertFile) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(relpSrvSetPrivKey(pSrv, (char*) inst->myPrivKeyFile) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + for(i = 0 ; i < inst->permittedPeers.nmemb ; ++i) { + relpSrvAddPermittedPeer(pSrv, (char*)inst->permittedPeers.name[i]); + } + } + CHKiRet(relpEngineListnerConstructFinalize(pRelpEngine, pSrv)); finalize_it: RETiRet; @@ -226,7 +350,7 @@ finalize_it: BEGINnewInpInst struct cnfparamvals *pvals; instanceConf_t *inst; - int i; + int i,j; CODESTARTnewInpInst DBGPRINTF("newInpInst (imrelp)\n"); @@ -249,6 +373,29 @@ CODESTARTnewInpInst continue; if(!strcmp(inppblk.descr[i].name, "port")) { inst->pszBindPort = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(inppblk.descr[i].name, "tls")) { + inst->bEnableTLS = (unsigned) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "tls.dhbits")) { + inst->dhBits = (unsigned) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "tls.prioritystring")) { + inst->pristring = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(inppblk.descr[i].name, "tls.authmode")) { + inst->authmode = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(inppblk.descr[i].name, "tls.compression")) { + inst->bEnableTLSZip = (unsigned) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "tls.cacert")) { + inst->caCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(inppblk.descr[i].name, "tls.mycert")) { + inst->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(inppblk.descr[i].name, "tls.myprivkey")) { + inst->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(inppblk.descr[i].name, "tls.permittedpeer")) { + inst->permittedPeers.nmemb = pvals[i].val.d.ar->nmemb; + CHKmalloc(inst->permittedPeers.name = + malloc(sizeof(uchar*) * inst->permittedPeers.nmemb)); + for(j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) { + inst->permittedPeers.name[j] = (uchar*)es_str2cstr(pvals[i].val.d.ar->arr[j], NULL); + } } else { dbgprintf("imrelp: program error, non-handled " "param '%s'\n", inppblk.descr[i].name); @@ -264,19 +411,58 @@ BEGINbeginCnfLoad CODESTARTbeginCnfLoad loadModConf = pModConf; pModConf->pConf = pConf; + pModConf->pszBindRuleset = NULL; + pModConf->pBindRuleset = NULL; /* init legacy config variables */ cs.pszBindRuleset = NULL; ENDbeginCnfLoad +BEGINsetModCnf + struct cnfparamvals *pvals = NULL; + int i; +CODESTARTsetModCnf + pvals = nvlstGetParams(lst, &modpblk, NULL); + if(pvals == NULL) { + errmsg.LogError(0, RS_RET_MISSING_CNFPARAMS, "error processing module " + "config parameters [module(...)]"); + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + + if(Debug) { + dbgprintf("module (global) param blk for imrelp:\n"); + cnfparamsPrint(&modpblk, pvals); + } + + for(i = 0 ; i < modpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(modpblk.descr[i].name, "ruleset")) { + loadModConf->pszBindRuleset = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else { + dbgprintf("imrelp: program error, non-handled " + "param '%s' in beginCnfLoad\n", modpblk.descr[i].name); + } + } +finalize_it: + if(pvals != NULL) + cnfparamvalsDestruct(pvals, &modpblk); +ENDsetModCnf + BEGINendCnfLoad CODESTARTendCnfLoad - if((cs.pszBindRuleset == NULL) || (cs.pszBindRuleset[0] == '\0')) { - loadModConf->pszBindRuleset = NULL; + if(loadModConf->pszBindRuleset == NULL) { + if((cs.pszBindRuleset == NULL) || (cs.pszBindRuleset[0] == '\0')) { + loadModConf->pszBindRuleset = NULL; + } else { + CHKmalloc(loadModConf->pszBindRuleset = ustrdup(cs.pszBindRuleset)); + } } else { - CHKmalloc(loadModConf->pszBindRuleset = ustrdup(cs.pszBindRuleset)); + if((cs.pszBindRuleset != NULL) && (cs.pszBindRuleset[0] != '\0')) { + errmsg.LogError(0, RS_RET_DUP_PARAM, "imrelp: warning: ruleset " + "set via legacy directive ignored"); + } } - loadModConf->pBindRuleset = NULL; finalize_it: free(cs.pszBindRuleset); loadModConf = NULL; /* done loading */ @@ -293,6 +479,7 @@ CODESTARTcheckCnf if(pModConf->pszBindRuleset == NULL) { pModConf->pBindRuleset = NULL; } else { + DBGPRINTF("imrelp: using ruleset '%s'\n", pModConf->pszBindRuleset); localRet = ruleset.GetRuleset(pModConf->pConf, &pRuleset, pModConf->pszBindRuleset); if(localRet == RS_RET_NOT_FOUND) { std_checkRuleset_genErrMsg(pModConf, NULL); @@ -323,13 +510,21 @@ ENDactivateCnf BEGINfreeCnf instanceConf_t *inst, *del; + int i; CODESTARTfreeCnf for(inst = pModConf->root ; inst != NULL ; ) { free(inst->pszBindPort); + free(inst->pristring); + free(inst->authmode); + statsobj.Destruct(&(inst->data.stats)); + for(i = 0 ; i < inst->permittedPeers.nmemb ; ++i) { + free(inst->permittedPeers.name[i]); + } del = inst; inst = inst->next; free(del); } + free(pModConf->pszBindRuleset); ENDfreeCnf /* This is used to terminate the plugin. Note that the signal handler blocks @@ -390,6 +585,7 @@ CODESTARTmodExit prop.Destruct(&pInputName); /* release objects we used */ + objRelease(statsobj, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); @@ -420,6 +616,7 @@ CODEqueryEtryPt_STD_IMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_PREPRIVDROP_QUERIES CODEqueryEtryPt_STD_CONF2_IMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_setModCnf_QUERIES CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt @@ -435,6 +632,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); CHKiRet(objUse(ruleset, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrelpserverbindruleset", 0, eCmdHdlrGetWord, diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index c503852c..dad09ab4 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -1284,6 +1284,8 @@ BEGINactivateCnfPrePrivDrop instanceConf_t *inst; CODESTARTactivateCnfPrePrivDrop runModConf = pModConf; + if(runModConf->bOmitLocalLogging && nfd == 1) + ABORT_FINALIZE(RS_RET_OK); for(inst = runModConf->root ; inst != NULL ; inst = inst->next) { addListner(inst); } @@ -1325,6 +1327,8 @@ BEGINrunInput #endif CODESTARTrunInput + if(runModConf->bOmitLocalLogging && nfd == 1) + ABORT_FINALIZE(RS_RET_OK); /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, * right into the sleep below. diff --git a/plugins/mmcount/Makefile.am b/plugins/mmcount/Makefile.am new file mode 100644 index 00000000..9c8c99db --- /dev/null +++ b/plugins/mmcount/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = mmcount.la + +mmcount_la_SOURCES = mmcount.c +mmcount_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +mmcount_la_LDFLAGS = -module -avoid-version +mmcount_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/mmcount/mmcount.c b/plugins/mmcount/mmcount.c new file mode 100644 index 00000000..56a4de55 --- /dev/null +++ b/plugins/mmcount/mmcount.c @@ -0,0 +1,342 @@ +/* mmcount.c + * count messages by priority or json property of given app-name. + * + * Copyright 2013 Red Hat Inc. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include <stdint.h> +#include <json/json.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "hashtable.h" + +#define JSON_COUNT_NAME "!mmcount" +#define SEVERITY_COUNT 8 + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP +MODULE_CNFNAME("mmcount") + + +DEFobjCurrIf(errmsg); +DEF_OMOD_STATIC_DATA + +/* config variables */ + +typedef struct _instanceData { + char *pszAppName; + int severity[SEVERITY_COUNT]; + char *pszKey; + char *pszValue; + int valueCounter; + struct hashtable *ht; +} instanceData; + +struct modConfData_s { + rsconf_t *pConf; /* our overall config object */ +}; +static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ +static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current exec process */ + + +/* tables for interfacing with the v6 config system */ +/* action (instance) parameters */ +static struct cnfparamdescr actpdescr[] = { + { "appname", eCmdHdlrGetWord, 0 }, + { "key", eCmdHdlrGetWord, 0 }, + { "value", eCmdHdlrGetWord, 0 }, +}; +static struct cnfparamblk actpblk = + { CNFPARAMBLK_VERSION, + sizeof(actpdescr)/sizeof(struct cnfparamdescr), + actpdescr + }; + +BEGINbeginCnfLoad +CODESTARTbeginCnfLoad + loadModConf = pModConf; + pModConf->pConf = pConf; +ENDbeginCnfLoad + +BEGINendCnfLoad +CODESTARTendCnfLoad +ENDendCnfLoad + +BEGINcheckCnf +CODESTARTcheckCnf +ENDcheckCnf + +BEGINactivateCnf +CODESTARTactivateCnf + runModConf = pModConf; +ENDactivateCnf + +BEGINfreeCnf +CODESTARTfreeCnf +ENDfreeCnf + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +static inline void +setInstParamDefaults(instanceData *pData) +{ + int i; + + pData->pszAppName = NULL; + for (i = 0; i < SEVERITY_COUNT; i++) + pData->severity[i] = 0; + pData->pszKey = NULL; + pData->pszValue = NULL; + pData->valueCounter = 0; + pData->ht = NULL; +} + +static unsigned int +hash_from_key_fn(void *k) +{ + return *(unsigned int *)k; +} + +static int +key_equals_fn(void *k1, void *k2) +{ + return (*(unsigned int *)k1 == *(unsigned int *)k2); +} + +BEGINnewActInst + struct cnfparamvals *pvals; + int i; +CODESTARTnewActInst + DBGPRINTF("newActInst (mmcount)\n"); + if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + + CODE_STD_STRING_REQUESTnewActInst(1) + CHKiRet(OMSRsetEntry(*ppOMSR, 0, NULL, OMSR_TPL_AS_MSG)); + CHKiRet(createInstance(&pData)); + setInstParamDefaults(pData); + + for(i = 0 ; i < actpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(actpblk.descr[i].name, "appname")) { + pData->pszAppName = es_str2cstr(pvals[i].val.d.estr, NULL); + continue; + } + if(!strcmp(actpblk.descr[i].name, "key")) { + pData->pszKey = es_str2cstr(pvals[i].val.d.estr, NULL); + continue; + } + if(!strcmp(actpblk.descr[i].name, "value")) { + pData->pszValue = es_str2cstr(pvals[i].val.d.estr, NULL); + continue; + } + dbgprintf("mmcount: program error, non-handled " + "param '%s'\n", actpblk.descr[i].name); + } + + if(pData->pszAppName == NULL) { + dbgprintf("mmcount: action requires a appname"); + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + + if(pData->pszKey != NULL && pData->pszValue == NULL) { + if(NULL == (pData->ht = create_hashtable(100, hash_from_key_fn, key_equals_fn, NULL))) { + DBGPRINTF("mmcount: error creating hash table!\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + } +CODE_STD_FINALIZERnewActInst + cnfparamvalsDestruct(pvals, &actpblk); +ENDnewActInst + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +static int * +getCounter(struct hashtable *ht, char *str) { + unsigned int key; + int *pCounter; + unsigned int *pKey; + + /* we dont store str as key, instead we store hash of the str + as key to reduce memory usage */ + key = hash_from_string(str); + pCounter = hashtable_search(ht, &key); + if(pCounter) { + return pCounter; + } + + /* counter is not found for the str, so add new entry and + return the counter */ + if(NULL == (pKey = (unsigned int*)malloc(sizeof(unsigned int)))) { + DBGPRINTF("mmcount: memory allocation for key failed\n"); + return NULL; + } + *pKey = key; + + if(NULL == (pCounter = (int*)malloc(sizeof(int)))) { + DBGPRINTF("mmcount: memory allocation for value failed\n"); + free(pKey); + return NULL; + } + *pCounter = 0; + + if(!hashtable_insert(ht, pKey, pCounter)) { + DBGPRINTF("mmcount: inserting element into hashtable failed\n"); + free(pKey); + free(pCounter); + return NULL; + } + return pCounter; +} + +BEGINdoAction + msg_t *pMsg; + char *appname; + struct json_object *json = NULL; + es_str_t *estr = NULL; + struct json_object *keyjson = NULL; + char *pszValue; + int *pCounter; +CODESTARTdoAction + pMsg = (msg_t*) ppString[0]; + appname = getAPPNAME(pMsg, LOCK_MUTEX); + + if(0 != strcmp(appname, pData->pszAppName)) { + /* we are not working for this appname. nothing to do */ + ABORT_FINALIZE(RS_RET_OK); + } + + if(!pData->pszKey) { + /* no key given for count, so we count severity */ + if(pMsg->iSeverity <= SEVERITY_COUNT) { + pData->severity[pMsg->iSeverity]++; + json = json_object_new_int(pData->severity[pMsg->iSeverity]); + } + ABORT_FINALIZE(RS_RET_OK); + } + + /* key is given, so get the property json */ + estr = es_newStrFromBuf(pData->pszKey, strlen(pData->pszKey)); + if(msgGetCEEPropJSON(pMsg, estr, &keyjson) != RS_RET_OK) { + /* key not found in the message. nothing to do */ + ABORT_FINALIZE(RS_RET_OK); + } + + /* key found, so get the value */ + pszValue = (char*)json_object_get_string(keyjson); + + if(pData->pszValue) { + /* value also given for count */ + if(!strcmp(pszValue, pData->pszValue)) { + /* count for (value and key and appname) matched */ + pData->valueCounter++; + json = json_object_new_int(pData->valueCounter); + } + ABORT_FINALIZE(RS_RET_OK); + } + + /* value is not given, so we count for each value of given key */ + pCounter = getCounter(pData->ht, pszValue); + if(pCounter) { + (*pCounter)++; + json = json_object_new_int(*pCounter); + } +finalize_it: + if(estr) { + es_deleteStr(estr); + } + + if(json) { + msgAddJSON(pMsg, (uchar *)JSON_COUNT_NAME, json); + } +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + if(strncmp((char*) p, ":mmcount:", sizeof(":mmcount:") - 1)) { + errmsg.LogError(0, RS_RET_LEGA_ACT_NOT_SUPPORTED, + "mmcount supports only v6+ config format, use: " + "action(type=\"mmcount\" ...)"); + } + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_QUERIES +ENDqueryEtryPt + + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + DBGPRINTF("mmcount: module compiled with rsyslog version %s.\n", VERSION); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +ENDmodInit diff --git a/plugins/mmfields/Makefile.am b/plugins/mmfields/Makefile.am new file mode 100644 index 00000000..08170d52 --- /dev/null +++ b/plugins/mmfields/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = mmfields.la + +mmfields_la_SOURCES = mmfields.c +mmfields_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +mmfields_la_LDFLAGS = -module -avoid-version +mmfields_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/mmfields/mmfields.c b/plugins/mmfields/mmfields.c new file mode 100644 index 00000000..fa7fa100 --- /dev/null +++ b/plugins/mmfields/mmfields.c @@ -0,0 +1,274 @@ +/* mmfields.c + * Parse all fields of the message into structured data inside the + * JSON tree. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include <stdint.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP +MODULE_CNFNAME("mmfields") + + +DEFobjCurrIf(errmsg); +DEF_OMOD_STATIC_DATA + +/* config variables */ + +/* define operation modes we have */ +#define SIMPLE_MODE 0 /* just overwrite */ +#define REWRITE_MODE 1 /* rewrite IP address, canoninized */ +typedef struct _instanceData { + char separator; + uchar *jsonRoot; /**< container where to store fields */ +} instanceData; + +struct modConfData_s { + rsconf_t *pConf; /* our overall config object */ +}; +static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ +static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current exec process */ + + +/* tables for interfacing with the v6 config system */ +/* action (instance) parameters */ +static struct cnfparamdescr actpdescr[] = { + { "separator", eCmdHdlrGetChar, 0 }, + { "jsonroot", eCmdHdlrString, 0 } +}; +static struct cnfparamblk actpblk = + { CNFPARAMBLK_VERSION, + sizeof(actpdescr)/sizeof(struct cnfparamdescr), + actpdescr + }; + +BEGINbeginCnfLoad +CODESTARTbeginCnfLoad + loadModConf = pModConf; + pModConf->pConf = pConf; +ENDbeginCnfLoad + +BEGINendCnfLoad +CODESTARTendCnfLoad +ENDendCnfLoad + +BEGINcheckCnf +CODESTARTcheckCnf +ENDcheckCnf + +BEGINactivateCnf +CODESTARTactivateCnf + runModConf = pModConf; +ENDactivateCnf + +BEGINfreeCnf +CODESTARTfreeCnf +ENDfreeCnf + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance + free(pData->jsonRoot); +ENDfreeInstance + + +static inline void +setInstParamDefaults(instanceData *pData) +{ + pData->separator = ','; + pData->jsonRoot = NULL; +} + +BEGINnewActInst + struct cnfparamvals *pvals; + int i; +CODESTARTnewActInst + DBGPRINTF("newActInst (mmfields)\n"); + if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + + CODE_STD_STRING_REQUESTnewActInst(1) + CHKiRet(OMSRsetEntry(*ppOMSR, 0, NULL, OMSR_TPL_AS_MSG)); + CHKiRet(createInstance(&pData)); + setInstParamDefaults(pData); + + for(i = 0 ; i < actpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(actpblk.descr[i].name, "separator")) { + pData->separator = es_getBufAddr(pvals[i].val.d.estr)[0]; + } else if(!strcmp(actpblk.descr[i].name, "jsonroot")) { + pData->jsonRoot = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else { + dbgprintf("mmfields: program error, non-handled " + "param '%s'\n", actpblk.descr[i].name); + } + } + if(pData->jsonRoot == NULL) { + CHKmalloc(pData->jsonRoot = (uchar*) strdup("!")); + } + +CODE_STD_FINALIZERnewActInst + cnfparamvalsDestruct(pvals, &actpblk); +ENDnewActInst + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + + +static inline rsRetVal +extractField(instanceData *pData, uchar *msgtext, int lenMsg, int *curridx, uchar *fieldbuf) +{ + int i, j; + DEFiRet; + i = *curridx; + j = 0; + while(i < lenMsg && msgtext[i] != pData->separator) { + fieldbuf[j++] = msgtext[i++]; + } + fieldbuf[j] = '\0'; + if(i < lenMsg) + ++i; + *curridx = i; + + RETiRet; +} + + +static inline rsRetVal +parse_fields(instanceData *pData, msg_t *pMsg, uchar *msgtext, int lenMsg) +{ + uchar fieldbuf[32*1024]; + uchar fieldname[512]; + struct json_object *json; + struct json_object *jval; + int field; + uchar *buf; + int currIdx = 0; + DEFiRet; + + if(lenMsg < (int) sizeof(fieldbuf)) { + buf = fieldbuf; + } else { + CHKmalloc(buf = malloc(lenMsg+1)); + } + + json = json_object_new_object(); + if(json == NULL) { + ABORT_FINALIZE(RS_RET_ERR); + } + field = 1; + while(currIdx < lenMsg) { + CHKiRet(extractField(pData, msgtext, lenMsg, &currIdx, buf)); + DBGPRINTF("mmfields: field %d: '%s'\n", field, buf); + snprintf((char*)fieldname, sizeof(fieldname), "f%d", field); + fieldname[sizeof(fieldname)-1] = '\0'; + jval = json_object_new_string((char*)fieldbuf); + json_object_object_add(json, (char*)fieldname, jval); + field++; + } + msgAddJSON(pMsg, pData->jsonRoot, json); +finalize_it: + RETiRet; +} + + +BEGINdoAction + msg_t *pMsg; + uchar *msg; + int lenMsg; +CODESTARTdoAction + pMsg = (msg_t*) ppString[0]; + lenMsg = getMSGLen(pMsg); + msg = getMSG(pMsg); + CHKiRet(parse_fields(pData, pMsg, msg, lenMsg)); +finalize_it: +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + if(strncmp((char*) p, ":mmfields:", sizeof(":mmfields:") - 1)) { + errmsg.LogError(0, RS_RET_LEGA_ACT_NOT_SUPPORTED, + "mmfields supports only v6+ config format, use: " + "action(type=\"mmfields\" ...)"); + } + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_QUERIES +ENDqueryEtryPt + + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + DBGPRINTF("mmfields: module compiled with rsyslog version %s.\n", VERSION); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +ENDmodInit diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index e425b428..7a6ef056 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -54,10 +54,12 @@ DEFobjCurrIf(errmsg) typedef struct _instanceData { uchar *szBinary; /* name of binary to call */ + char **aParams; /* Optional Parameters for binary command */ uchar *tplName; /* assigned output template */ - pid_t pid; /* pid of currently running process */ - int fdPipe; /* file descriptor to write to */ + pid_t pid; /* pid of currently running process */ + int fdPipe; /* file descriptor to write to */ int bIsRunning; /* is binary currently running? 0-no, 1-yes */ + int iParams; /* Holds the count of parameters if set*/ } instanceData; typedef struct configSettings_s { @@ -98,9 +100,16 @@ ENDisCompatibleWithFeature BEGINfreeInstance + int i; CODESTARTfreeInstance if(pData->szBinary != NULL) free(pData->szBinary); + if(pData->aParams != NULL) { + for (i = 0; i < pData->iParams; i++) { + free(pData->aParams[i]); + } + free(pData->aParams); + } ENDfreeInstance @@ -120,9 +129,8 @@ ENDtryResume static void execBinary(instanceData *pData, int fdStdin) { - int i; + int i, iRet; struct sigaction sigAct; - char *newargv[] = { NULL }; char *newenviron[] = { NULL }; assert(pData != NULL); @@ -134,7 +142,7 @@ static void execBinary(instanceData *pData, int fdStdin) * gets some more widespread use... */ } - //fclose(stdout); + /*fclose(stdout);*/ /* we close all file handles as we fork soon * Is there a better way to do this? - mail me! rgerhards@adiscon.com @@ -154,11 +162,11 @@ static void execBinary(instanceData *pData, int fdStdin) alarm(0); /* finally exec child */ - execve((char*)pData->szBinary, newargv, newenviron); - /* switch to? - execlp((char*)program, (char*) program, (char*)arg, NULL); - */ - + iRet = execve((char*)pData->szBinary, pData->aParams, newenviron); + if (iRet == -1) { + dbgprintf("omprog: failed to execute binary '%s' with return code: %d\n", pData->szBinary, errno); + } + /* we should never reach this point, but if we do, we terminate */ exit(1); } @@ -180,7 +188,7 @@ openPipe(instanceData *pData) ABORT_FINALIZE(RS_RET_ERR_CREAT_PIPE); } - DBGPRINTF("executing program '%s'\n", pData->szBinary); + DBGPRINTF("omprog: executing program '%s' with '%d' parameters\n", pData->szBinary, pData->iParams); /* NO OUTPUT AFTER FORK! */ @@ -198,7 +206,7 @@ openPipe(instanceData *pData) /*NO CODE HERE - WILL NEVER BE REACHED!*/ } - DBGPRINTF("child has pid %d\n", (int) cpid); + DBGPRINTF("omprog: child has pid %d\n", (int) cpid); pData->fdPipe = pipefd[1]; pData->pid = cpid; close(pipefd[0]); @@ -223,11 +231,11 @@ cleanup(instanceData *pData) ret = waitpid(pData->pid, &status, 0); if(ret != pData->pid) { /* if waitpid() fails, we can not do much - try to ignore it... */ - DBGPRINTF("waitpid() returned state %d[%s], future malfunction may happen\n", ret, + DBGPRINTF("omprog: waitpid() returned state %d[%s], future malfunction may happen\n", ret, rs_strerror_r(errno, errStr, sizeof(errStr))); } else { /* check if we should print out some diagnostic information */ - DBGPRINTF("waitpid status return for program '%s': %2.2x\n", + DBGPRINTF("omprog: waitpid status return for program '%s': %2.2x\n", pData->szBinary, status); if(WIFEXITED(status)) { errmsg.LogError(0, NO_ERRCODE, "program '%s' exited normally, state %d", @@ -282,13 +290,13 @@ writePipe(instanceData *pData, uchar *szMsg) if(lenWritten == -1) { switch(errno) { case EPIPE: - DBGPRINTF("Program '%s' terminated, trying to restart\n", + DBGPRINTF("omprog: Program '%s' terminated, trying to restart\n", pData->szBinary); CHKiRet(cleanup(pData)); CHKiRet(tryRestart(pData)); break; default: - DBGPRINTF("error %d writing to pipe: %s\n", errno, + DBGPRINTF("omprog: error %d writing to pipe: %s\n", errno, rs_strerror_r(errno, errStr, sizeof(errStr))); ABORT_FINALIZE(RS_RET_ERR_WRITE_PIPE); break; @@ -321,13 +329,23 @@ static inline void setInstParamDefaults(instanceData *pData) { pData->szBinary = NULL; + pData->aParams = NULL; + pData->iParams = 0; pData->fdPipe = -1; pData->bIsRunning = 0; } BEGINnewActInst struct cnfparamvals *pvals; + sbool bInQuotes; int i; + int iPrm; + unsigned char *c; + es_size_t iCnt; + es_size_t iStr; + es_str_t *estrBinary; + es_str_t *estrParams; + es_str_t *estrTmp; CODESTARTnewActInst if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) { ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); @@ -341,23 +359,97 @@ CODESTARTnewActInst if(!pvals[i].bUsed) continue; if(!strcmp(actpblk.descr[i].name, "binary")) { - pData->szBinary = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + estrBinary = pvals[i].val.d.estr; + estrParams = NULL; + + /* Search for space */ + c = es_getBufAddr(pvals[i].val.d.estr); + iCnt = 0; + while(iCnt < es_strlen(pvals[i].val.d.estr) ) { + if (c[iCnt] == ' ') { + /* Split binary name from parameters */ + estrBinary = es_newStrFromSubStr ( pvals[i].val.d.estr, 0, iCnt ); + estrParams = es_newStrFromSubStr ( pvals[i].val.d.estr, iCnt+1, es_strlen(pvals[i].val.d.estr)); + break; + } + iCnt++; + } + /* Assign binary and params */ + pData->szBinary = (uchar*)es_str2cstr(estrBinary, NULL); + dbgprintf("omprog: szBinary = '%s'\n", pData->szBinary); + /* Check for Params! */ + if (estrParams != NULL) { + dbgprintf("omprog: szParams = '%s'\n", es_str2cstr(estrParams, NULL) ); + + /* Count parameters if set */ + c = es_getBufAddr(estrParams); /* Reset to beginning */ + pData->iParams = 2; /* Set default to 2, first parameter for binary and second parameter at least from config*/ + iCnt = 0; + while(iCnt < es_strlen(estrParams) ) { + if (c[iCnt] == ' ' && c[iCnt-1] != '\\') + pData->iParams++; + iCnt++; + } + dbgprintf("omprog: iParams = '%d'\n", pData->iParams); + + /* Create argv Array */ + CHKmalloc(pData->aParams = malloc( (pData->iParams+1) * sizeof(char*))); /* One more for first param */ + + /* Second Loop, create parameter array*/ + c = es_getBufAddr(estrParams); /* Reset to beginning */ + iCnt = iStr = iPrm = 0; + estrTmp = NULL; + bInQuotes = FALSE; + /* Set first parameter to binary */ + pData->aParams[iPrm] = strdup((char*)pData->szBinary); + dbgprintf("omprog: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); + iPrm++; + while(iCnt < es_strlen(estrParams) ) { + if ( c[iCnt] == ' ' && !bInQuotes ) { + /* Copy into Param Array! */ + estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr); + } + else if ( iCnt+1 >= es_strlen(estrParams) ) { + /* Copy rest of string into Param Array! */ + estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr+1); + } + else if (c[iCnt] == '"') { + /* switch inQuotes Mode */ + bInQuotes = !bInQuotes; + } + + if ( estrTmp != NULL ) { + pData->aParams[iPrm] = es_str2cstr(estrTmp, NULL); + iStr = iCnt+1; /* Set new start */ + dbgprintf("omprog: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); + es_deleteStr( estrTmp ); + estrTmp = NULL; + iPrm++; + } + + /*Next char*/ + iCnt++; + } + /* NULL last parameter! */ + pData->aParams[iPrm] = NULL; + + } } else if(!strcmp(actpblk.descr[i].name, "template")) { pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else { - dbgprintf("omprog: program error, non-handled " - "param '%s'\n", actpblk.descr[i].name); + dbgprintf("omprog: program error, non-handled param '%s'\n", actpblk.descr[i].name); } } if(pData->tplName == NULL) { - CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*) "RSYSLOG_FileFormat", + CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*) strdup("RSYSLOG_FileFormat"), OMSR_NO_RQD_TPL_OPTS)); } else { CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*) strdup((char*) pData->tplName), OMSR_NO_RQD_TPL_OPTS)); } + CODE_STD_FINALIZERnewActInst cnfparamvalsDestruct(pvals, &actpblk); ENDnewActInst diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 3e355464..34511e46 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -55,6 +55,9 @@ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) +#define DFLT_ENABLE_TLS 0 +#define DFLT_ENABLE_TLSZIP 0 + static relpEngine_t *pRelpEngine; /* our relp engine */ typedef struct _instanceData { @@ -62,9 +65,24 @@ typedef struct _instanceData { uchar *port; int bInitialConnect; /* is this the initial connection request of our module? (0-no, 1-yes) */ int bIsConnected; /* currently connected to server? 0 - no, 1 - yes */ + int sizeWindow; /**< the RELP window size - 0=use default */ unsigned timeout; + unsigned rebindInterval; + unsigned nSent; relpClt_t *pRelpClt; /* relp client for this instance */ + sbool bEnableTLS; + sbool bEnableTLSZip; + sbool bHadAuthFail; /**< set on auth failure, will cause retry to disable action */ + uchar *pristring; /* GnuTLS priority string (NULL if not to be provided) */ + uchar *authmode; + uchar *caCertFile; + uchar *myCertFile; + uchar *myPrivKeyFile; uchar *tplName; + struct { + int nmemb; + uchar **name; + } permittedPeers; } instanceData; typedef struct configSettings_s { @@ -77,7 +95,17 @@ static configSettings_t __attribute__((unused)) cs; /* action (instance) parameters */ static struct cnfparamdescr actpdescr[] = { { "target", eCmdHdlrGetWord, 1 }, + { "tls", eCmdHdlrBinary, 0 }, + { "tls.compression", eCmdHdlrBinary, 0 }, + { "tls.prioritystring", eCmdHdlrString, 0 }, + { "tls.cacert", eCmdHdlrString, 0 }, + { "tls.mycert", eCmdHdlrString, 0 }, + { "tls.myprivkey", eCmdHdlrString, 0 }, + { "tls.authmode", eCmdHdlrString, 0 }, + { "tls.permittedpeer", eCmdHdlrArray, 0 }, { "port", eCmdHdlrGetWord, 0 }, + { "rebindinterval", eCmdHdlrInt, 0 }, + { "windowsize", eCmdHdlrInt, 0 }, { "timeout", eCmdHdlrInt, 0 }, { "template", eCmdHdlrGetWord, 0 } }; @@ -104,32 +132,111 @@ static uchar *getRelpPt(instanceData *pData) return(pData->port); } +static void +onErr(void *pUsr, char *objinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) +{ + instanceData *pData = (instanceData*) pUsr; + errmsg.LogError(0, RS_RET_RELP_AUTH_FAIL, "omrelp[%s:%s]: error '%s', object " + " '%s' - action may not work as intended", + pData->target, pData->port, errmesg, objinfo); +} + +static void +onGenericErr(char *objinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) +{ + errmsg.LogError(0, RS_RET_RELP_ERR, "omrelp: librelp error '%s', object " + "'%s' - action may not work as intended", + errmesg, objinfo); +} + +static void +onAuthErr(void *pUsr, char *authinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) +{ + instanceData *pData = (instanceData*) pUsr; + errmsg.LogError(0, RS_RET_RELP_AUTH_FAIL, "omrelp[%s:%s]: authentication error '%s', peer " + "is '%s' - DISABLING action", pData->target, pData->port, errmesg, authinfo); + pData->bHadAuthFail = 1; +} + static inline rsRetVal doCreateRelpClient(instanceData *pData) { + int i; DEFiRet; if(relpEngineCltConstruct(pRelpEngine, &pData->pRelpClt) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); if(relpCltSetTimeout(pData->pRelpClt, pData->timeout) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); + if(relpCltSetWindowSize(pData->pRelpClt, pData->sizeWindow) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(relpCltSetUsrPtr(pData->pRelpClt, pData) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(pData->bEnableTLS) { + if(relpCltEnableTLS(pData->pRelpClt) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(pData->bEnableTLSZip) { + if(relpCltEnableTLSZip(pData->pRelpClt) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + } + if(relpCltSetGnuTLSPriString(pData->pRelpClt, (char*) pData->pristring) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(relpCltSetAuthMode(pData->pRelpClt, (char*) pData->authmode) != RELP_RET_OK) { + errmsg.LogError(0, RS_RET_RELP_ERR, + "omrelp: invalid auth mode '%s'\n", pData->authmode); + ABORT_FINALIZE(RS_RET_RELP_ERR); + } + if(relpCltSetCACert(pData->pRelpClt, (char*) pData->caCertFile) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(relpCltSetOwnCert(pData->pRelpClt, (char*) pData->myCertFile) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + if(relpCltSetPrivKey(pData->pRelpClt, (char*) pData->myPrivKeyFile) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + for(i = 0 ; i < pData->permittedPeers.nmemb ; ++i) { + relpCltAddPermittedPeer(pData->pRelpClt, (char*)pData->permittedPeers.name[i]); + } + } + if(glbl.GetSourceIPofLocalClient() == NULL) { /* ar Do we have a client IP set? */ + if(relpCltSetClientIP(pData->pRelpClt, glbl.GetSourceIPofLocalClient()) != RELP_RET_OK) + ABORT_FINALIZE(RS_RET_RELP_ERR); + } + pData->bInitialConnect = 1; + pData->nSent = 0; finalize_it: RETiRet; } - BEGINcreateInstance CODESTARTcreateInstance - pData->bInitialConnect = 1; + pData->sizeWindow = 0; pData->timeout = 90; + pData->rebindInterval = 0; + pData->bEnableTLS = DFLT_ENABLE_TLS; + pData->bEnableTLSZip = DFLT_ENABLE_TLSZIP; + pData->bHadAuthFail = 0; + pData->pristring = NULL; + pData->authmode = NULL; + pData->caCertFile = NULL; + pData->myCertFile = NULL; + pData->myPrivKeyFile = NULL; + pData->permittedPeers.nmemb = 0; ENDcreateInstance BEGINfreeInstance + int i; CODESTARTfreeInstance if(pData->pRelpClt != NULL) relpEngineCltDestruct(pRelpEngine, &pData->pRelpClt); free(pData->target); free(pData->port); free(pData->tplName); + free(pData->pristring); + free(pData->authmode); + free(pData->caCertFile); + free(pData->myCertFile); + free(pData->myPrivKeyFile); + for(i = 0 ; i < pData->permittedPeers.nmemb ; ++i) { + free(pData->permittedPeers.name[i]); + } ENDfreeInstance static inline void @@ -139,12 +246,22 @@ setInstParamDefaults(instanceData *pData) pData->port = NULL; pData->tplName = NULL; pData->timeout = 90; + pData->sizeWindow = 0; + pData->rebindInterval = 0; + pData->bEnableTLS = DFLT_ENABLE_TLS; + pData->bEnableTLSZip = DFLT_ENABLE_TLSZIP; + pData->pristring = NULL; + pData->authmode = NULL; + pData->caCertFile = NULL; + pData->myCertFile = NULL; + pData->myPrivKeyFile = NULL; + pData->permittedPeers.nmemb = 0; } BEGINnewActInst struct cnfparamvals *pvals; - int i; + int i,j; CODESTARTnewActInst if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) { ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); @@ -164,6 +281,31 @@ CODESTARTnewActInst pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "timeout")) { pData->timeout = (unsigned) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "rebindinterval")) { + pData->rebindInterval = (unsigned) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "windowsize")) { + pData->sizeWindow = (int) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "tls")) { + pData->bEnableTLS = (unsigned) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "tls.compression")) { + pData->bEnableTLSZip = (unsigned) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "tls.prioritystring")) { + pData->pristring = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "tls.cacert")) { + pData->caCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "tls.mycert")) { + pData->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "tls.myprivkey")) { + pData->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "tls.authmode")) { + pData->authmode = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "tls.permittedpeer")) { + pData->permittedPeers.nmemb = pvals[i].val.d.ar->nmemb; + CHKmalloc(pData->permittedPeers.name = + malloc(sizeof(uchar*) * pData->permittedPeers.nmemb)); + for(j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) { + pData->permittedPeers.name[j] = (uchar*)es_str2cstr(pvals[i].val.d.ar->arr[j], NULL); + } } else { dbgprintf("omrelp: program error, non-handled " "param '%s'\n", actpblk.descr[i].name); @@ -230,9 +372,34 @@ static rsRetVal doConnect(instanceData *pData) BEGINtryResume CODESTARTtryResume + if(pData->bHadAuthFail) { + ABORT_FINALIZE(RS_RET_DISABLE_ACTION); + } iRet = doConnect(pData); +finalize_it: ENDtryResume +static inline rsRetVal +doRebind(instanceData *pData) +{ + DEFiRet; + DBGPRINTF("omrelp: destructing relp client due to rebindInterval\n"); + CHKiRet(relpEngineCltDestruct(pRelpEngine, &pData->pRelpClt)); + pData->bIsConnected = 0; + CHKiRet(doCreateRelpClient(pData)); +finalize_it: + RETiRet; +} + +BEGINbeginTransaction +CODESTARTbeginTransaction +dbgprintf("omrelp: beginTransaction\n"); + if(!pData->bIsConnected) { + CHKiRet(doConnect(pData)); + } + relpCltHintBurstBegin(pData->pRelpClt); +finalize_it: +ENDbeginTransaction BEGINdoAction uchar *pMsg; /* temporary buffering */ @@ -248,7 +415,7 @@ CODESTARTdoAction pMsg = ppString[0]; lenMsg = strlen((char*) pMsg); /* TODO: don't we get this? */ - /* TODO: think about handling oversize messages! */ + /* we need to truncate oversize msgs - no way around that... */ if((int) lenMsg > glbl.GetMaxLine()) lenMsg = glbl.GetMaxLine(); @@ -257,13 +424,33 @@ CODESTARTdoAction if(ret != RELP_RET_OK) { /* error! */ dbgprintf("error forwarding via relp, suspending\n"); - iRet = RS_RET_SUSPENDED; + ABORT_FINALIZE(RS_RET_SUSPENDED); } + if(pData->rebindInterval != 0 && + (++pData->nSent >= pData->rebindInterval)) { + doRebind(pData); + } finalize_it: + if(pData->bHadAuthFail) + iRet = RS_RET_DISABLE_ACTION; + if(iRet == RS_RET_OK) { + /* we mimic non-commit, as otherwise our endTransaction handler + * will not get called. While this is not 100% correct, the worst + * that can happen is some message duplication, something that + * rsyslog generally accepts and prefers over message loss. + */ + iRet = RS_RET_PREVIOUS_COMMITTED; + } ENDdoAction +BEGINendTransaction +CODESTARTendTransaction + dbgprintf("omrelp: endTransaction\n"); + relpCltHintBurstEnd(pData->pRelpClt); +ENDendTransaction + BEGINparseSelectorAct uchar *q; int i; @@ -329,7 +516,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) ++p; } - /* TODO: make this if go away! */ if(*p == ';') { *p = '\0'; /* trick to obtain hostname (later)! */ CHKmalloc(pData->target = ustrdup(q)); @@ -362,6 +548,7 @@ CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES CODEqueryEtryPt_SetShutdownImmdtPtr ENDqueryEtryPt @@ -374,12 +561,12 @@ CODEmodInit_QueryRegCFSLineHdlr /* create our relp engine */ CHKiRet(relpEngineConstruct(&pRelpEngine)); CHKiRet(relpEngineSetDbgprint(pRelpEngine, dbgprintf)); + CHKiRet(relpEngineSetOnAuthErr(pRelpEngine, onAuthErr)); + CHKiRet(relpEngineSetOnGenericErr(pRelpEngine, onGenericErr)); + CHKiRet(relpEngineSetOnErr(pRelpEngine, onErr)); CHKiRet(relpEngineSetEnableCmd(pRelpEngine, (uchar*) "syslog", eRelpCmdState_Required)); /* tell which objects we need */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDmodInit - -/* vim:set ai: - */ diff --git a/runtime/cryprov.h b/runtime/cryprov.h index 8496b745..5690904d 100644 --- a/runtime/cryprov.h +++ b/runtime/cryprov.h @@ -26,14 +26,25 @@ #include <gcrypt.h> +/* we unfortunately need to have two different param names depending on the + * context in which parameters are set. Other than (re/over)engineering the core + * interface, we just define some values to keep track of that. + */ +#define CRYPROV_PARAMTYPE_REGULAR 0 +#define CRYPROV_PARAMTYPE_DISK 1 + /* interface */ BEGINinterface(cryprov) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Construct)(void *ppThis); - rsRetVal (*SetCnfParam)(void *ppThis, struct nvlst *lst); + rsRetVal (*SetCnfParam)(void *ppThis, struct nvlst *lst, int paramType); rsRetVal (*Destruct)(void *ppThis); - rsRetVal (*OnFileOpen)(void *pThis, uchar *fn, void *pFileInstData); + rsRetVal (*OnFileOpen)(void *pThis, uchar *fn, void *pFileInstData, char openMode); rsRetVal (*Encrypt)(void *pFileInstData, uchar *buf, size_t *lenBuf); + rsRetVal (*Decrypt)(void *pFileInstData, uchar *buf, size_t *lenBuf); rsRetVal (*OnFileClose)(void *pFileInstData, off64_t offsLogfile); + rsRetVal (*DeleteStateFiles)(uchar *logfn); + rsRetVal (*GetBytesLeftInBlock)(void *pFileInstData, ssize_t *left); + void (*SetDeleteOnClose)(void *pFileInstData, int val); ENDinterface(cryprov) -#define cryprovCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define cryprovCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ #endif /* #ifndef INCLUDED_CRYPROV_H */ diff --git a/runtime/glbl.c b/runtime/glbl.c index b3fe3a1d..ccb978ba 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -89,6 +89,7 @@ static DEF_ATOMIC_HELPER_MUT(mutTerminateInputs); #ifdef USE_UNLIMITED_SELECT static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */ #endif +static uchar *SourceIPofLocalClient = NULL; /* [ar] Source IP for local client to be used on multihomed host */ /* tables for interfacing with the v6 config system */ @@ -478,6 +479,23 @@ GetDfltNetstrmDrvrCertFile(void) } +/* [ar] Source IP for local client to be used on multihomed host */ +static rsRetVal +SetSourceIPofLocalClient(uchar *newname) +{ + if(SourceIPofLocalClient != NULL) { + free(SourceIPofLocalClient); } + SourceIPofLocalClient = newname; + return RS_RET_OK; +} + +static uchar* +GetSourceIPofLocalClient(void) +{ + return(SourceIPofLocalClient); +} + + /* queryInterface function * rgerhards, 2008-02-21 */ @@ -498,6 +516,8 @@ CODESTARTobjQueryInterface(glbl) pIf->GetLocalHostIP = GetLocalHostIP; pIf->SetGlobalInputTermination = SetGlobalInputTermination; pIf->GetGlobalInputTermState = GetGlobalInputTermState; + pIf->GetSourceIPofLocalClient = GetSourceIPofLocalClient; /* [ar] */ + pIf->SetSourceIPofLocalClient = SetSourceIPofLocalClient; /* [ar] */ #define SIMP_PROP(name) \ pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; diff --git a/runtime/glbl.h b/runtime/glbl.h index e95e48f7..2c7f3b31 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -81,6 +81,8 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ /* next change is v9! */ /* v8 - 2012-03-21 */ prop_t* (*GetLocalHostIP)(void); + uchar* (*GetSourceIPofLocalClient)(void); /* [ar] */ + rsRetVal (*SetSourceIPofLocalClient)(uchar*); /* [ar] */ #undef SIMP_PROP ENDinterface(glbl) #define glblCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ diff --git a/runtime/libgcry.c b/runtime/libgcry.c index 51c10af4..4772cf47 100644 --- a/runtime/libgcry.c +++ b/runtime/libgcry.c @@ -49,8 +49,12 @@ #include <errno.h> #include "rsyslog.h" +#include "srUtils.h" #include "libgcry.h" +#define READBUF_SIZE 4096 /* size of the read buffer */ + +static rsRetVal rsgcryBlkBegin(gcryfile gf); static rsRetVal eiWriteRec(gcryfile gf, char *recHdr, size_t lenRecHdr, char *buf, size_t lenBuf) @@ -90,19 +94,66 @@ finalize_it: RETiRet; } +static rsRetVal +eiRead(gcryfile gf) +{ + ssize_t nRead; + DEFiRet; + + if(gf->readBuf == NULL) { + CHKmalloc(gf->readBuf = malloc(READBUF_SIZE)); + } + + nRead = read(gf->fd, gf->readBuf, READBUF_SIZE); + if(nRead <= 0) { /* TODO: provide specific EOF case? */ + ABORT_FINALIZE(RS_RET_ERR); + } + gf->readBufMaxIdx = (int16_t) nRead; + gf->readBufIdx = 0; + +finalize_it: + RETiRet; +} + + +/* returns EOF on any kind of error */ +static int +eiReadChar(gcryfile gf) +{ + int c; + + if(gf->readBufIdx >= gf->readBufMaxIdx) { + if(eiRead(gf) != RS_RET_OK) { + c = EOF; + goto finalize_it; + } + } + c = gf->readBuf[gf->readBufIdx++]; +finalize_it: + return c; +} + static rsRetVal eiCheckFiletype(gcryfile gf) { char hdrBuf[128]; size_t toRead, didRead; + sbool bNeedClose = 0; DEFiRet; - CHKiRet(eiOpenRead(gf)); + if(gf->fd == -1) { + bNeedClose = 1; + CHKiRet(eiOpenRead(gf)); + } + if(Debug) memset(hdrBuf, 0, sizeof(hdrBuf)); /* for dbgprintf below! */ toRead = sizeof("FILETYPE:")-1 + sizeof(RSGCRY_FILETYPE_NAME)-1 + 1; didRead = read(gf->fd, hdrBuf, toRead); - close(gf->fd); + if(bNeedClose) { + close(gf->fd); + gf->fd = -1; + } DBGPRINTF("eiCheckFiletype read %d bytes: '%s'\n", didRead, hdrBuf); if( didRead != toRead || strncmp(hdrBuf, "FILETYPE:" RSGCRY_FILETYPE_NAME "\n", toRead)) @@ -111,6 +162,98 @@ finalize_it: RETiRet; } +/* rectype/value must be EIF_MAX_*_LEN+1 long! + * returns 0 on success or something else on error/EOF + */ +static rsRetVal +eiGetRecord(gcryfile gf, char *rectype, char *value) +{ + unsigned short i, j; + int c; + DEFiRet; + + c = eiReadChar(gf); + if(c == EOF) { ABORT_FINALIZE(RS_RET_NO_DATA); } + for(i = 0 ; i < EIF_MAX_RECTYPE_LEN ; ++i) { + if(c == ':' || c == EOF) + break; + rectype[i] = c; + c = eiReadChar(gf); + } + if(c != ':') { ABORT_FINALIZE(RS_RET_ERR); } + rectype[i] = '\0'; + j = 0; + for(++i ; i < EIF_MAX_VALUE_LEN ; ++i, ++j) { + c = eiReadChar(gf); + if(c == '\n' || c == EOF) + break; + value[j] = c; + } + if(c != '\n') { ABORT_FINALIZE(RS_RET_ERR); } + value[j] = '\0'; +finalize_it: + RETiRet; +} + +static rsRetVal +eiGetIV(gcryfile gf, uchar *iv, size_t leniv) +{ + char rectype[EIF_MAX_RECTYPE_LEN+1]; + char value[EIF_MAX_VALUE_LEN+1]; + size_t valueLen; + unsigned short i, j; + unsigned char nibble; + DEFiRet; + + CHKiRet(eiGetRecord(gf, rectype, value)); + if(strcmp(rectype, "IV")) { + DBGPRINTF("no IV record found when expected, record type " + "seen is '%s'\n", rectype); + ABORT_FINALIZE(RS_RET_ERR); + } + valueLen = strlen(value); + if(valueLen/2 != leniv) { + DBGPRINTF("length of IV is %d, expected %d\n", + valueLen/2, leniv); + ABORT_FINALIZE(RS_RET_ERR); + } + + for(i = j = 0 ; i < valueLen ; ++i) { + if(value[i] >= '0' && value[i] <= '9') + nibble = value[i] - '0'; + else if(value[i] >= 'a' && value[i] <= 'f') + nibble = value[i] - 'a' + 10; + else { + DBGPRINTF("invalid IV '%s'\n", value); + ABORT_FINALIZE(RS_RET_ERR); + } + if(i % 2 == 0) + iv[j] = nibble << 4; + else + iv[j++] |= nibble; + } +finalize_it: + RETiRet; +} + +static rsRetVal +eiGetEND(gcryfile gf, off64_t *offs) +{ + char rectype[EIF_MAX_RECTYPE_LEN+1]; + char value[EIF_MAX_VALUE_LEN+1]; + DEFiRet; + + CHKiRet(eiGetRecord(gf, rectype, value)); + if(strcmp(rectype, "END")) { + DBGPRINTF("no END record found when expected, record type " + "seen is '%s'\n", rectype); + ABORT_FINALIZE(RS_RET_ERR); + } + *offs = atoll(value); +finalize_it: + RETiRet; +} + static rsRetVal eiOpenAppend(gcryfile gf) { @@ -177,13 +320,55 @@ eiClose(gcryfile gf, off64_t offsLogfile) size_t len; if(gf->fd == -1) return; - /* 2^64 is 20 digits, so the snprintf buffer is large enough */ - len = snprintf(offs, sizeof(offs), "%lld", offsLogfile); - eiWriteRec(gf, "END:", 4, offs, len); + if(gf->openMode == 'w') { + /* 2^64 is 20 digits, so the snprintf buffer is large enough */ + len = snprintf(offs, sizeof(offs), "%lld", offsLogfile); + eiWriteRec(gf, "END:", 4, offs, len); + } + gcry_cipher_close(gf->chd); + free(gf->readBuf); close(gf->fd); + gf->fd = -1; DBGPRINTF("encryption info file %s: closed\n", gf->eiName); } +/* this returns the number of bytes left inside the block or -1, if the block + * size is unbounded. The function automatically handles end-of-block and begins + * to read the next block in this case. + */ +rsRetVal +gcryfileGetBytesLeftInBlock(gcryfile gf, ssize_t *left) +{ + DEFiRet; + if(gf->bytesToBlkEnd == 0) { + DBGPRINTF("libgcry: end of current crypto block\n"); + gcry_cipher_close(gf->chd); + CHKiRet(rsgcryBlkBegin(gf)); + } + *left = gf->bytesToBlkEnd; +finalize_it: + // TODO: remove once this code is sufficiently well-proven + DBGPRINTF("gcryfileGetBytesLeftInBlock returns %lld, iRet %d\n", (long long) *left, iRet); + RETiRet; +} + +/* this is a special functon for use by the rsyslog disk queue subsystem. It + * needs to have the capability to delete state when a queue file is rolled + * over. This simply generates the file name and deletes it. It must take care + * of "all" state files, which currently happens to be a single one. + */ +rsRetVal +gcryfileDeleteState(uchar *logfn) +{ + char fn[MAXFNAME+1]; + DEFiRet; + snprintf(fn, sizeof(fn), "%s%s", logfn, ENCINFO_SUFFIX); + fn[MAXFNAME] = '\0'; /* be on save side */ + DBGPRINTF("crypto provider deletes state file '%s' on request\n", fn); + unlink(fn); + RETiRet; +} + static rsRetVal gcryfileConstruct(gcryctx ctx, gcryfile *pgf, uchar *logfn) { @@ -193,6 +378,7 @@ gcryfileConstruct(gcryctx ctx, gcryfile *pgf, uchar *logfn) CHKmalloc(gf = calloc(1, sizeof(struct gcryfile_s))); gf->ctx = ctx; + gf->fd = -1; snprintf(fn, sizeof(fn), "%s%s", logfn, ENCINFO_SUFFIX); fn[MAXFNAME] = '\0'; /* be on save side */ gf->eiName = (uchar*) strdup(fn); @@ -219,7 +405,12 @@ gcryfileDestruct(gcryfile gf, off64_t offsLogfile) if(gf == NULL) goto done; + DBGPRINTF("libgcry: close file %s\n", gf->eiName); eiClose(gf, offsLogfile); + if(gf->bDeleteOnClose) { + DBGPRINTF("unlink file '%s' due to bDeleteOnClose set\n", gf->eiName); + unlink((char*)gf->eiName); + } free(gf->eiName); free(gf); done: return r; @@ -246,13 +437,13 @@ addPadding(gcryfile pF, uchar *buf, size_t *plen) } static inline void -removePadding(char *buf, size_t *plen) +removePadding(uchar *buf, size_t *plen) { unsigned len = (unsigned) *plen; unsigned iSrc, iDst; - char *frstNUL; + uchar *frstNUL; - frstNUL = strchr(buf, 0x00); + frstNUL = (uchar*)strchr((char*)buf, 0x00); if(frstNUL == NULL) goto done; iDst = iSrc = frstNUL - buf; @@ -343,53 +534,123 @@ seedIV(gcryfile gf, uchar **iv) } } -rsRetVal -rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, uchar *fname) +static inline rsRetVal +readIV(gcryfile gf, uchar **iv) { - gcry_error_t gcryError; - gcryfile gf = NULL; - uchar *iv = NULL; + rsRetVal localRet; DEFiRet; - CHKiRet(gcryfileConstruct(ctx, &gf, fname)); + if(gf->fd == -1) { + while(gf->fd == -1) { + localRet = eiOpenRead(gf); + if(localRet == RS_RET_EI_NO_EXISTS) { + /* wait until it is created */ + srSleep(0, 10000); + } else { + CHKiRet(localRet); + } + } + CHKiRet(eiCheckFiletype(gf)); + } + *iv = malloc(gf->blkLength); /* do NOT zero-out! */ + CHKiRet(eiGetIV(gf, *iv, (size_t) gf->blkLength)); +finalize_it: + RETiRet; +} + +/* this tries to read the END record. HOWEVER, no such record may be + * present, which is the case if we handle a currently-written to queue + * file. On the other hand, the queue file may contain multiple blocks. So + * what we do is try to see if there is a block end or not - and set the + * status accordingly. Note that once we found no end-of-block, we will never + * retry. This is because that case can never happen under current queue + * implementations. -- gerhards, 2013-05-16 + */ +static inline rsRetVal +readBlkEnd(gcryfile gf) +{ + off64_t blkEnd; + DEFiRet; + + iRet = eiGetEND(gf, &blkEnd); + if(iRet == RS_RET_OK) { + gf->bytesToBlkEnd = (ssize_t) blkEnd; + } else if(iRet == RS_RET_NO_DATA) { + gf->bytesToBlkEnd = -1; + } else { + FINALIZE; + } + +finalize_it: + RETiRet; +} - gf->blkLength = gcry_cipher_get_algo_blklen(ctx->algo); - gcryError = gcry_cipher_open(&gf->chd, ctx->algo, ctx->mode, 0); +/* Read the block begin metadata and set our state variables accordingly. Can also + * be used to init the first block in write case. + */ +static rsRetVal +rsgcryBlkBegin(gcryfile gf) +{ + gcry_error_t gcryError; + uchar *iv = NULL; + DEFiRet; + + gcryError = gcry_cipher_open(&gf->chd, gf->ctx->algo, gf->ctx->mode, 0); if (gcryError) { - dbgprintf("gcry_cipher_open failed: %s/%s\n", - gcry_strsource(gcryError), - gcry_strerror(gcryError)); + DBGPRINTF("gcry_cipher_open failed: %s/%s\n", + gcry_strsource(gcryError), gcry_strerror(gcryError)); ABORT_FINALIZE(RS_RET_ERR); } gcryError = gcry_cipher_setkey(gf->chd, gf->ctx->key, gf->ctx->keyLen); if (gcryError) { - dbgprintf("gcry_cipher_setkey failed: %s/%s\n", - gcry_strsource(gcryError), - gcry_strerror(gcryError)); + DBGPRINTF("gcry_cipher_setkey failed: %s/%s\n", + gcry_strsource(gcryError), gcry_strerror(gcryError)); ABORT_FINALIZE(RS_RET_ERR); } - seedIV(gf, &iv); + if(gf->openMode == 'r') { + readIV(gf, &iv); + readBlkEnd(gf); + } else { + seedIV(gf, &iv); + } + gcryError = gcry_cipher_setiv(gf->chd, iv, gf->blkLength); if (gcryError) { - dbgprintf("gcry_cipher_setiv failed: %s/%s\n", - gcry_strsource(gcryError), - gcry_strerror(gcryError)); + DBGPRINTF("gcry_cipher_setiv failed: %s/%s\n", + gcry_strsource(gcryError), gcry_strerror(gcryError)); ABORT_FINALIZE(RS_RET_ERR); } - CHKiRet(eiOpenAppend(gf)); - CHKiRet(eiWriteIV(gf, iv)); - *pgf = gf; + + if(gf->openMode == 'w') { + CHKiRet(eiOpenAppend(gf)); + CHKiRet(eiWriteIV(gf, iv)); + } finalize_it: free(iv); + RETiRet; +} + +rsRetVal +rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, uchar *fname, char openMode) +{ + gcryfile gf = NULL; + DEFiRet; + + CHKiRet(gcryfileConstruct(ctx, &gf, fname)); + gf->openMode = openMode; + gf->blkLength = gcry_cipher_get_algo_blklen(ctx->algo); + CHKiRet(rsgcryBlkBegin(gf)); + *pgf = gf; +finalize_it: if(iRet != RS_RET_OK && gf != NULL) gcryfileDestruct(gf, -1); RETiRet; } -int +rsRetVal rsgcryEncrypt(gcryfile pF, uchar *buf, size_t *len) { int gcryError; @@ -410,6 +671,35 @@ finalize_it: RETiRet; } +/* TODO: handle multiple blocks + * test-read END record; if present, store offset, else unbounded (current active block) + * when decrypting, check if bound is reached. If yes, split into two blocks, get new IV for + * second one. + */ +rsRetVal +rsgcryDecrypt(gcryfile pF, uchar *buf, size_t *len) +{ + gcry_error_t gcryError; + DEFiRet; + + if(pF->bytesToBlkEnd != -1) + pF->bytesToBlkEnd -= *len; + gcryError = gcry_cipher_decrypt(pF->chd, buf, *len, NULL, 0); + if(gcryError) { + DBGPRINTF("gcry_cipher_decrypt failed: %s/%s\n", + gcry_strsource(gcryError), + gcry_strerror(gcryError)); + ABORT_FINALIZE(RS_RET_ERR); + } + removePadding(buf, len); + // TODO: remove dbgprintf once things are sufficently stable -- rgerhards, 2013-05-16 + dbgprintf("libgcry: decrypted, bytesToBlkEnd %lld, buffer is now '%50.50s'\n", (long long) pF->bytesToBlkEnd, buf); + +finalize_it: + RETiRet; +} + + /* module-init dummy for potential later use */ int diff --git a/runtime/libgcry.h b/runtime/libgcry.h index b77b0f9e..2f700554 100644 --- a/runtime/libgcry.h +++ b/runtime/libgcry.h @@ -38,7 +38,15 @@ struct gcryfile_s { size_t blkLength; /* size of low-level crypto block */ uchar *eiName; /* name of .encinfo file */ int fd; /* descriptor of .encinfo file (-1 if not open) */ + char openMode; /* 'r': read, 'w': write */ gcryctx ctx; + uchar *readBuf; + int16_t readBufIdx; + int16_t readBufMaxIdx; + int8_t bDeleteOnClose; /* for queue support, similar to stream subsys */ + ssize_t bytesToBlkEnd; /* number of bytes remaining in current crypto block + -1 means -> no end (still being writen to, queue files), + 0 means -> end of block, new one must be started. */ }; int gcryGetKeyFromFile(char *fn, char **key, unsigned *keylen); @@ -50,8 +58,12 @@ rsRetVal rsgcrySetAlgo(gcryctx ctx, uchar *modename); gcryctx gcryCtxNew(void); void rsgcryCtxDel(gcryctx ctx); int gcryfileDestruct(gcryfile gf, off64_t offsLogfile); -rsRetVal rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, uchar *fname); -int rsgcryEncrypt(gcryfile pF, uchar *buf, size_t *len); +rsRetVal rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, uchar *fname, char openMode); +rsRetVal rsgcryEncrypt(gcryfile pF, uchar *buf, size_t *len); +rsRetVal rsgcryDecrypt(gcryfile pF, uchar *buf, size_t *len); +int gcryGetKeyFromProg(char *cmd, char **key, unsigned *keylen); +rsRetVal gcryfileDeleteState(uchar *fn); +rsRetVal gcryfileGetBytesLeftInBlock(gcryfile gf, ssize_t *left); /* error states */ #define RSGCRYE_EI_OPEN 1 /* error opening .encinfo file */ @@ -62,6 +74,14 @@ int rsgcryEncrypt(gcryfile pF, uchar *buf, size_t *len); #define RSGCRY_FILETYPE_NAME "rsyslog-enrcyption-info" #define ENCINFO_SUFFIX ".encinfo" +/* Note: gf may validly be NULL, e.g. if file has not yet been opened! */ +static inline void +gcryfileSetDeleteOnClose(gcryfile gf, int val) +{ + if(gf != NULL) + gf->bDeleteOnClose = val; +} + static inline int rsgcryAlgoname2Algo(char *algoname) { if(!strcmp((char*)algoname, "3DES")) return GCRY_CIPHER_3DES; diff --git a/runtime/lmcry_gcry.c b/runtime/lmcry_gcry.c index 0a9b94bc..9a0c0072 100644 --- a/runtime/lmcry_gcry.c +++ b/runtime/lmcry_gcry.c @@ -43,17 +43,30 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) /* tables for interfacing with the v6 config system */ -static struct cnfparamdescr cnfpdescr[] = { +static struct cnfparamdescr cnfpdescrRegular[] = { { "cry.key", eCmdHdlrGetWord, 0 }, { "cry.keyfile", eCmdHdlrGetWord, 0 }, { "cry.keyprogram", eCmdHdlrGetWord, 0 }, { "cry.mode", eCmdHdlrGetWord, 0 }, /* CBC, ECB, etc */ { "cry.algo", eCmdHdlrGetWord, 0 } }; -static struct cnfparamblk pblk = +static struct cnfparamblk pblkRegular = { CNFPARAMBLK_VERSION, - sizeof(cnfpdescr)/sizeof(struct cnfparamdescr), - cnfpdescr + sizeof(cnfpdescrRegular)/sizeof(struct cnfparamdescr), + cnfpdescrRegular + }; + +static struct cnfparamdescr cnfpdescrQueue[] = { + { "queue.cry.key", eCmdHdlrGetWord, 0 }, + { "queue.cry.keyfile", eCmdHdlrGetWord, 0 }, + { "queue.cry.keyprogram", eCmdHdlrGetWord, 0 }, + { "queue.cry.mode", eCmdHdlrGetWord, 0 }, /* CBC, ECB, etc */ + { "queue.cry.algo", eCmdHdlrGetWord, 0 } +}; +static struct cnfparamblk pblkQueue = + { CNFPARAMBLK_VERSION, + sizeof(cnfpdescrQueue)/sizeof(struct cnfparamdescr), + cnfpdescrQueue }; @@ -85,7 +98,7 @@ ENDobjDestruct(lmcry_gcry) * Defaults are expected to have been set during construction. */ static rsRetVal -SetCnfParam(void *pT, struct nvlst *lst) +SetCnfParam(void *pT, struct nvlst *lst, int paramType) { lmcry_gcry_t *pThis = (lmcry_gcry_t*) pT; int i, r; @@ -97,34 +110,41 @@ SetCnfParam(void *pT, struct nvlst *lst) uchar *mode = NULL; int nKeys; /* number of keys (actually methods) specified */ struct cnfparamvals *pvals; + struct cnfparamblk *pblk; DEFiRet; + pblk = (paramType == CRYPROV_PARAMTYPE_REGULAR ) ? &pblkRegular : &pblkQueue; nKeys = 0; - pvals = nvlstGetParams(lst, &pblk, NULL); + pvals = nvlstGetParams(lst, pblk, NULL); if(Debug) { dbgprintf("param blk in lmcry_gcry:\n"); - cnfparamsPrint(&pblk, pvals); + cnfparamsPrint(pblk, pvals); } - for(i = 0 ; i < pblk.nParams ; ++i) { + for(i = 0 ; i < pblk->nParams ; ++i) { if(!pvals[i].bUsed) continue; - if(!strcmp(pblk.descr[i].name, "cry.key")) { + if(!strcmp(pblk->descr[i].name, "cry.key") || + !strcmp(pblk->descr[i].name, "queue.cry.key")) { key = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); ++nKeys; - } else if(!strcmp(pblk.descr[i].name, "cry.keyfile")) { + } else if(!strcmp(pblk->descr[i].name, "cry.keyfile") || + !strcmp(pblk->descr[i].name, "queue.cry.keyfile")) { keyfile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); ++nKeys; - } else if(!strcmp(pblk.descr[i].name, "cry.keyprogram")) { + } else if(!strcmp(pblk->descr[i].name, "cry.keyprogram") || + !strcmp(pblk->descr[i].name, "queue.cry.keyprogram")) { keyprogram = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); ++nKeys; - } else if(!strcmp(pblk.descr[i].name, "cry.mode")) { + } else if(!strcmp(pblk->descr[i].name, "cry.mode") || + !strcmp(pblk->descr[i].name, "queue.cry.mode")) { mode = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); - } else if(!strcmp(pblk.descr[i].name, "cry.algo")) { + } else if(!strcmp(pblk->descr[i].name, "cry.algo") || + !strcmp(pblk->descr[i].name, "queue.cry.algo")) { algo = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else { DBGPRINTF("lmcry_gcry: program error, non-handled " - "param '%s'\n", pblk.descr[i].name); + "param '%s'\n", pblk->descr[i].name); } } if(algo != NULL) { @@ -177,7 +197,7 @@ SetCnfParam(void *pT, struct nvlst *lst) ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } - cnfparamvalsDestruct(pvals, &pblk); + cnfparamvalsDestruct(pvals, pblk); if(key != NULL) { memset(key, 0, strlen((char*)key)); free(key); @@ -189,15 +209,33 @@ finalize_it: RETiRet; } +static void +SetDeleteOnClose(void *pF, int val) +{ + gcryfileSetDeleteOnClose(pF, val); +} + +static rsRetVal +GetBytesLeftInBlock(void *pF, ssize_t *left) +{ + return gcryfileGetBytesLeftInBlock((gcryfile) pF, left); +} static rsRetVal -OnFileOpen(void *pT, uchar *fn, void *pGF) +DeleteStateFiles(uchar *logfn) +{ + return gcryfileDeleteState(logfn); +} + +static rsRetVal +OnFileOpen(void *pT, uchar *fn, void *pGF, char openMode) { lmcry_gcry_t *pThis = (lmcry_gcry_t*) pT; gcryfile *pgf = (gcryfile*) pGF; DEFiRet; + DBGPRINTF("lmcry_gcry: open file '%s', mode '%c'\n", fn, openMode); - CHKiRet(rsgcryInitCrypt(pThis->ctx, pgf, fn)); + CHKiRet(rsgcryInitCrypt(pThis->ctx, pgf, fn, openMode)); finalize_it: /* TODO: enable this error message (need to cleanup loop first ;)) errmsg.LogError(0, iRet, "Encryption Provider" @@ -207,6 +245,16 @@ finalize_it: } static rsRetVal +Decrypt(void *pF, uchar *rec, size_t *lenRec) +{ + DEFiRet; + iRet = rsgcryDecrypt(pF, rec, lenRec); + + RETiRet; +} + + +static rsRetVal Encrypt(void *pF, uchar *rec, size_t *lenRec) { DEFiRet; @@ -231,10 +279,14 @@ CODESTARTobjQueryInterface(lmcry_gcry) } pIf->Construct = (rsRetVal(*)(void*)) lmcry_gcryConstruct; pIf->SetCnfParam = SetCnfParam; + pIf->SetDeleteOnClose = SetDeleteOnClose; pIf->Destruct = (rsRetVal(*)(void*)) lmcry_gcryDestruct; pIf->OnFileOpen = OnFileOpen; pIf->Encrypt = Encrypt; + pIf->Decrypt = Decrypt; pIf->OnFileClose = OnFileClose; + pIf->DeleteStateFiles = DeleteStateFiles; + pIf->GetBytesLeftInBlock = GetBytesLeftInBlock; finalize_it: ENDobjQueryInterface(lmcry_gcry) diff --git a/runtime/queue.c b/runtime/queue.c index 85b1e45b..699e2a66 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -12,7 +12,7 @@ * function names - this makes it really hard to read and does not provide much * benefit, at least I (now) think so... * - * Copyright 2008-2011 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -118,6 +118,7 @@ static struct cnfparamdescr cnfpdescr[] = { { "queue.dequeueslowdown", eCmdHdlrInt, 0 }, { "queue.dequeuetimebegin", eCmdHdlrInt, 0 }, { "queue.dequeuetimeend", eCmdHdlrInt, 0 }, + { "queue.cry.provider", eCmdHdlrGetWord, 0 } }; static struct cnfparamblk pblk = { CNFPARAMBLK_VERSION, @@ -776,11 +777,19 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - /* create a duplicate for the read "pointer". */ CHKiRet(strm.Dup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq)); CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */ CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); + /* if we use a crypto provider, we need to amend the objects with it's info */ + if(pThis->useCryprov) { + CHKiRet(strm.Setcryprov(pThis->tVars.disk.pWrite, &pThis->cryprov)); + CHKiRet(strm.SetcryprovData(pThis->tVars.disk.pWrite, pThis->cryprovData)); + CHKiRet(strm.Setcryprov(pThis->tVars.disk.pReadDeq, &pThis->cryprov)); + CHKiRet(strm.SetcryprovData(pThis->tVars.disk.pReadDeq, pThis->cryprovData)); + CHKiRet(strm.Setcryprov(pThis->tVars.disk.pReadDel, &pThis->cryprov)); + CHKiRet(strm.SetcryprovData(pThis->tVars.disk.pReadDel, pThis->cryprovData)); + } CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite)); CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDel)); @@ -834,6 +843,10 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); + if(pThis->useCryprov) { + CHKiRet(strm.Setcryprov(pThis->tVars.disk.pWrite, &pThis->cryprov)); + CHKiRet(strm.SetcryprovData(pThis->tVars.disk.pWrite, pThis->cryprovData)); + } CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDeq)); @@ -842,6 +855,10 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); + if(pThis->useCryprov) { + CHKiRet(strm.Setcryprov(pThis->tVars.disk.pReadDeq, &pThis->cryprov)); + CHKiRet(strm.SetcryprovData(pThis->tVars.disk.pReadDeq, pThis->cryprovData)); + } CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDel)); @@ -851,6 +868,10 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ)); CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR)); + if(pThis->useCryprov) { + CHKiRet(strm.Setcryprov(pThis->tVars.disk.pReadDel, &pThis->cryprov)); + CHKiRet(strm.SetcryprovData(pThis->tVars.disk.pReadDel, pThis->cryprovData)); + } CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDel)); CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); @@ -1320,6 +1341,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; pThis->nLogDeq = 0; + pThis->useCryprov = 0; pThis->iMaxQueueSize = iMaxQueueSize; pThis->pConsumer = pConsumer; pThis->iNumWorkerThreads = iWorkerThreads; @@ -2389,6 +2411,13 @@ CODESTARTobjDestruct(qqueue) free(pThis->pszFilePrefix); free(pThis->pszSpoolDir); + if(pThis->useCryprov) { + pThis->cryprov.Destruct(&pThis->cryprovData); + obj.ReleaseObj(__FILE__, pThis->cryprovNameFull+2, pThis->cryprovNameFull, + (void*) &pThis->cryprov); + free(pThis->cryprovName); + free(pThis->cryprovNameFull); + } /* some queues do not provide stats and thus have no statsobj! */ if(pThis->statsobj != NULL) @@ -2672,27 +2701,67 @@ finalize_it: } -/* take v6 config list and extract the queue params out of it. Hand the - * param values back to the caller. Caller is responsible for destructing - * them when no longer needed. Caller can use this param block to configure - * all parameters for a newly created queue with one call to qqueueSetParams(). - * rgerhards, 2011-07-22 +/* are any queue params set at all? 1 - yes, 0 - no + * We need to evaluate the param block for this function, which is somewhat + * inefficient. HOWEVER, this is only done during config load, so we really + * don't care... -- rgerhards, 2013-05-10 */ -rsRetVal -qqueueDoCnfParams(struct nvlst *lst, struct cnfparamvals **ppvals) +int +queueCnfParamsSet(struct nvlst *lst) { - *ppvals = nvlstGetParams(lst, &pblk, NULL); - return RS_RET_OK; + int r; + struct cnfparamvals *pvals; + + pvals = nvlstGetParams(lst, &pblk, NULL); + r = cnfparamvalsIsSet(&pblk, pvals); + cnfparamvalsDestruct(pvals, &pblk); + return r; } -/* are any queue params set at all? 1 - yes, 0 - no */ -int -queueCnfParamsSet(struct cnfparamvals *pvals) +static inline rsRetVal +initCryprov(qqueue_t *pThis, struct nvlst *lst) { - return cnfparamvalsIsSet(&pblk, pvals); -} + uchar szDrvrName[1024]; + DEFiRet; + if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmcry_%s", pThis->cryprovName) + == sizeof(szDrvrName)) { + errmsg.LogError(0, RS_RET_ERR, "queue: crypto provider " + "name is too long: '%s' - encryption disabled", + pThis->cryprovName); + ABORT_FINALIZE(RS_RET_ERR); + } + pThis->cryprovNameFull = ustrdup(szDrvrName); + + pThis->cryprov.ifVersion = cryprovCURR_IF_VERSION; + /* The pDrvrName+2 below is a hack to obtain the object name. It + * safes us to have yet another variable with the name without "lm" in + * front of it. If we change the module load interface, we may re-think + * about this hack, but for the time being it is efficient and clean enough. + */ + if(obj.UseObj(__FILE__, szDrvrName, szDrvrName, (void*) &pThis->cryprov) + != RS_RET_OK) { + errmsg.LogError(0, RS_RET_LOAD_ERROR, "queue: could not load " + "crypto provider '%s' - encryption disabled", + szDrvrName); + ABORT_FINALIZE(RS_RET_CRYPROV_ERR); + } + + if(pThis->cryprov.Construct(&pThis->cryprovData) != RS_RET_OK) { + errmsg.LogError(0, RS_RET_CRYPROV_ERR, "queue: error constructing " + "crypto provider %s dataset - encryption disabled", + szDrvrName); + ABORT_FINALIZE(RS_RET_CRYPROV_ERR); + } + CHKiRet(pThis->cryprov.SetCnfParam(pThis->cryprovData, lst, CRYPROV_PARAMTYPE_DISK)); + + dbgprintf("loaded crypto provider %s, data instance at %p\n", + szDrvrName, pThis->cryprovData); + pThis->useCryprov = 1; +finalize_it: + RETiRet; +} /* apply all params from param block to queue. Must be called before * finalizing. This supports the v6 config system. Defaults were already @@ -2700,15 +2769,24 @@ queueCnfParamsSet(struct cnfparamvals *pvals) * function. */ rsRetVal -qqueueApplyCnfParam(qqueue_t *pThis, struct cnfparamvals *pvals) +qqueueApplyCnfParam(qqueue_t *pThis, struct nvlst *lst) { int i; + struct cnfparamvals *pvals; + + pvals = nvlstGetParams(lst, &pblk, NULL); + if(Debug) { + dbgprintf("queue param blk:\n"); + cnfparamsPrint(&pblk, pvals); + } for(i = 0 ; i < pblk.nParams ; ++i) { if(!pvals[i].bUsed) continue; if(!strcmp(pblk.descr[i].name, "queue.filename")) { pThis->pszFilePrefix = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); pThis->lenFilePrefix = es_strlen(pvals[i].val.d.estr); + } else if(!strcmp(pblk.descr[i].name, "queue.cry.provider")) { + pThis->cryprovName = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(pblk.descr[i].name, "queue.size")) { pThis->iMaxQueueSize = pvals[i].val.d.n; } else if(!strcmp(pblk.descr[i].name, "queue.dequeuebatchsize")) { @@ -2760,12 +2838,27 @@ qqueueApplyCnfParam(qqueue_t *pThis, struct cnfparamvals *pvals) "param '%s'\n", pblk.descr[i].name); } } - if(pThis->qType == QUEUETYPE_DISK && pThis->pszFilePrefix == NULL) { - errmsg.LogError(0, RS_RET_QUEUE_DISK_NO_FN, "error on queue '%s', disk mode selected, but " - "no queue file name given; queue type changed to 'linkedList'", + if(pThis->qType == QUEUETYPE_DISK) { + if(pThis->pszFilePrefix == NULL) { + errmsg.LogError(0, RS_RET_QUEUE_DISK_NO_FN, "error on queue '%s', disk mode selected, but " + "no queue file name given; queue type changed to 'linkedList'", + obj.GetName((obj_t*) pThis)); + pThis->qType = QUEUETYPE_LINKEDLIST; + } + } + + if(pThis->pszFilePrefix == NULL && pThis->cryprovName != NULL) { + errmsg.LogError(0, RS_RET_QUEUE_CRY_DISK_ONLY, "error on queue '%s', crypto provider can " + "only be set for disk or disk assisted queue - ignored", obj.GetName((obj_t*) pThis)); - pThis->qType = QUEUETYPE_LINKEDLIST; + free(pThis->cryprovName); + pThis->cryprovName = NULL; + } + + if(pThis->cryprovName != NULL) { + initCryprov(pThis, lst); } + cnfparamvalsDestruct(pvals, &pblk); return RS_RET_OK; } diff --git a/runtime/queue.h b/runtime/queue.h index 886fac8d..844523ad 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -30,6 +30,7 @@ #include "batch.h" #include "stream.h" #include "statsobj.h" +#include "cryprov.h" /* support for the toDelete list */ typedef struct toDeleteLst_s toDeleteLst_t; @@ -168,6 +169,11 @@ struct queue_s { strm_t *pReadDel; /* current file for deleting */ } disk; } tVars; + sbool useCryprov; /* quicker than checkig ptr (1 vs 8 bytes!) */ + uchar *cryprovName; /* crypto provider to use */ + cryprov_if_t cryprov; /* ptr to crypto provider interface */ + void *cryprovData; /* opaque data ptr for provider use */ + uchar *cryprovNameFull;/* full internal crypto provider name */ DEF_ATOMIC_HELPER_MUT(mutQueueSize); DEF_ATOMIC_HELPER_MUT(mutLogDeq); /* for statistics subsystem */ @@ -197,9 +203,8 @@ rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefi rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*)); rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch); -rsRetVal qqueueDoCnfParams(struct nvlst *lst, struct cnfparamvals **ppvals); -int queueCnfParamsSet(struct cnfparamvals *pvals); -rsRetVal qqueueApplyCnfParam(qqueue_t *pThis, struct cnfparamvals *pvals); +int queueCnfParamsSet(struct nvlst *lst); +rsRetVal qqueueApplyCnfParam(qqueue_t *pThis, struct nvlst *lst); void qqueueSetDefaultsRulesetQueue(qqueue_t *pThis); void qqueueSetDefaultsActionQueue(qqueue_t *pThis); void qqueueDbgPrint(qqueue_t *pThis); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 47b34783..63112627 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -413,6 +413,10 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_CRY_INVLD_ALGO = -2326,/**< user specified invalid (unkonwn) crypto algorithm */ RS_RET_CRY_INVLD_MODE = -2327,/**< user specified invalid (unkonwn) crypto mode */ RS_RET_QUEUE_DISK_NO_FN = -2328,/**< disk queue configured, but filename not set */ + /* up to 2350 reserved for 7.4 */ + RS_RET_QUEUE_CRY_DISK_ONLY = -2351,/**< crypto provider only supported for disk-associated queues */ + RS_RET_NO_DATA = -2352,/**< file has no data; more a state than a real error */ + RS_RET_RELP_AUTH_FAIL = -2353,/**< RELP peer authentication failed */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c index e3348938..aacbdf57 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -928,7 +928,6 @@ rsRetVal rulesetProcessCnf(struct cnfobj *o) { struct cnfparamvals *pvals; - struct cnfparamvals *queueParams; rsRetVal localRet; uchar *rsName = NULL; uchar *parserName; @@ -974,11 +973,10 @@ rulesetProcessCnf(struct cnfobj *o) } /* pick up ruleset queue parameters */ - qqueueDoCnfParams(o->nvlst, &queueParams); - if(queueCnfParamsSet(queueParams)) { + if(queueCnfParamsSet(o->nvlst)) { rsname = (pRuleset->pszName == NULL) ? (uchar*) "[ruleset]" : pRuleset->pszName; DBGPRINTF("adding a ruleset-specific \"main\" queue for ruleset '%s'\n", rsname); - CHKiRet(createMainQueue(&pRuleset->pQueue, rsname, queueParams)); + CHKiRet(createMainQueue(&pRuleset->pQueue, rsname, o->nvlst)); } finalize_it: diff --git a/runtime/srutils.c b/runtime/srutils.c index 6a509b4a..8eb2459c 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -86,6 +86,7 @@ syslogName_t syslogFacNames[] = { {"mark", LOG_MARK}, /* INTERNAL */ {"news", LOG_NEWS}, {"security", LOG_AUTH}, /* DEPRECATED */ + {"bsd_security", (13<<3) }, /* BSD-specific, unfortunatly with duplicate name... */ {"syslog", LOG_SYSLOG}, {"user", LOG_USER}, {"uucp", LOG_UUCP}, @@ -95,6 +96,7 @@ syslogName_t syslogFacNames[] = { #if defined(LOG_AUDIT) {"audit", LOG_AUDIT}, #endif + {"console", (14 << 3)}, /* BSD-specific priority */ {"local0", LOG_LOCAL0}, {"local1", LOG_LOCAL1}, {"local2", LOG_LOCAL2}, diff --git a/runtime/stream.c b/runtime/stream.c index 54210e40..1af5b974 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -256,7 +256,9 @@ doPhysOpen(strm_t *pThis) if(pThis->cryprov != NULL) { CHKiRet(pThis->cryprov->OnFileOpen(pThis->cryprovData, - pThis->pszCurrFName, &pThis->cryprovFileData)); + pThis->pszCurrFName, &pThis->cryprovFileData, + (pThis->tOperationsMode == STREAMMODE_READ) ? 'r' : 'w')); + pThis->cryprov->SetDeleteOnClose(pThis->cryprovFileData, pThis->bDeleteOnClose); } finalize_it: RETiRet; @@ -404,6 +406,12 @@ static rsRetVal strmCloseFile(strm_t *pThis) } } + /* if we have a signature provider, we must make sure that the crypto + * state files are opened and proper close processing happens. */ + if(pThis->cryprov != NULL && pThis->fd == -1) { + strmOpenFile(pThis); + } + /* the file may already be closed (or never have opened), so guard * against this. -- rgerhards, 2010-03-19 */ @@ -550,11 +558,14 @@ finalize_it: * rgerhards, 2008-02-13 */ static rsRetVal -strmReadBuf(strm_t *pThis) +strmReadBuf(strm_t *pThis, int *padBytes) { DEFiRet; int bRun; long iLenRead; + size_t actualDataLen; + size_t toRead; + ssize_t bytesLeft; ISOBJ_TYPE_assert(pThis, strm); /* We need to try read at least twice because we may run into EOF and need to switch files. */ @@ -565,13 +576,35 @@ strmReadBuf(strm_t *pThis) * rgerhards, 2008-02-13 */ CHKiRet(strmOpenFile(pThis)); - iLenRead = read(pThis->fd, pThis->pIOBuf, pThis->sIOBufSize); + if(pThis->cryprov == NULL) { + toRead = pThis->sIOBufSize; + } else { + CHKiRet(pThis->cryprov->GetBytesLeftInBlock(pThis->cryprovFileData, &bytesLeft)); + if(bytesLeft == -1 || bytesLeft > (ssize_t) pThis->sIOBufSize) { + toRead = pThis->sIOBufSize; + } else { + toRead = (size_t) bytesLeft; + } + } + iLenRead = read(pThis->fd, pThis->pIOBuf, toRead); DBGOPRINT((obj_t*) pThis, "file %d read %ld bytes\n", pThis->fd, iLenRead); + /* end crypto */ if(iLenRead == 0) { CHKiRet(strmHandleEOF(pThis)); } else if(iLenRead < 0) ABORT_FINALIZE(RS_RET_IO_ERROR); else { /* good read */ + /* here we place our crypto interface */ + if(pThis->cryprov != NULL) { + actualDataLen = iLenRead; + pThis->cryprov->Decrypt(pThis->cryprovFileData, pThis->pIOBuf, &actualDataLen); + *padBytes = iLenRead - actualDataLen; + iLenRead = actualDataLen; + DBGOPRINT((obj_t*) pThis, "encrypted file %d pad bytes %d, actual " + "data %ld\n", pThis->fd, *padBytes, iLenRead); + } else { + *padBytes = 0; + } pThis->iBufPtrMax = iLenRead; bRun = 0; /* exit loop */ } @@ -593,6 +626,7 @@ finalize_it: */ static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) { + int padBytes = 0; /* in crypto mode, we may have some padding (non-data) bytes */ DEFiRet; ASSERT(pThis != NULL); @@ -608,8 +642,9 @@ static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) /* do we need to obtain a new buffer? */ if(pThis->iBufPtr >= pThis->iBufPtrMax) { - CHKiRet(strmReadBuf(pThis)); + CHKiRet(strmReadBuf(pThis, &padBytes)); } + pThis->iCurrOffs += padBytes; /* if we reach this point, we have data available in the buffer */ @@ -1454,6 +1489,8 @@ strmMultiFileSeek(strm_t *pThis, int FNum, off64_t offs, off64_t *bytesDel) "deleting '%s' (%lld bytes)\n", pThis->iCurrFNum, FNum, pThis->pszCurrFName, (long long) *bytesDel); unlink((char*)pThis->pszCurrFName); + if(pThis->cryprov != NULL) + pThis->cryprov->DeleteStateFiles(pThis->pszCurrFName); free(pThis->pszCurrFName); pThis->pszCurrFName = NULL; pThis->iCurrFNum = FNum; @@ -1467,17 +1504,31 @@ finalize_it: } - /* seek to current offset. This is primarily a helper to readjust the OS file * pointer after a strm object has been deserialized. */ static rsRetVal strmSeekCurrOffs(strm_t *pThis) { + off64_t targetOffs; + uchar c; DEFiRet; ISOBJ_TYPE_assert(pThis, strm); - iRet = strmSeek(pThis, pThis->iCurrOffs); + if(pThis->cryprov == NULL || pThis->tOperationsMode != STREAMMODE_READ) { + iRet = strmSeek(pThis, pThis->iCurrOffs); + FINALIZE; + } + + /* As the cryprov may use CBC or similiar things, we need to read skip data */ + targetOffs = pThis->iCurrOffs; + pThis->iCurrOffs = 0; + DBGOPRINT((obj_t*) pThis, "encrypted, doing skip read of %lld bytes\n", + (long long) targetOffs); + while(targetOffs != pThis->iCurrOffs) { + CHKiRet(strmReadChar(pThis, &c)); + } +finalize_it: RETiRet; } @@ -1604,7 +1655,6 @@ finalize_it: /* property set methods */ /* simple ones first */ -DEFpropSetMeth(strm, bDeleteOnClose, int) DEFpropSetMeth(strm, iMaxFileSize, int) DEFpropSetMeth(strm, iFileNumDigits, int) DEFpropSetMeth(strm, tOperationsMode, int) @@ -1620,6 +1670,15 @@ DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) DEFpropSetMeth(strm, cryprov, cryprov_if_t*) DEFpropSetMeth(strm, cryprovData, void*) +static rsRetVal strmSetbDeleteOnClose(strm_t *pThis, int val) +{ + pThis->bDeleteOnClose = val; + if(pThis->cryprov != NULL) { + pThis->cryprov->SetDeleteOnClose(pThis->cryprovFileData, pThis->bDeleteOnClose); + } + return RS_RET_OK; +} + static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { pThis->iMaxFiles = iNewVal; diff --git a/tools/omfile.c b/tools/omfile.c index ba9f7f70..2ebb7df9 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -155,7 +155,6 @@ typedef struct _instanceData { uchar *cryprovName; /* crypto provider */ uchar *cryprovNameFull;/* full internal crypto provider name */ void *cryprovData; /* opaque data ptr for provider use */ - void *cryprovFileData;/* opaque data ptr for file instance */ cryprov_if_t cryprov; /* ptr to crypto provider interface */ sbool useCryprov; /* quicker than checkig ptr (1 vs 8 bytes!) */ int iCurrElt; /* currently active cache element (-1 = none) */ @@ -1089,7 +1088,7 @@ initCryprov(instanceData *pData, struct nvlst *lst) szDrvrName); ABORT_FINALIZE(RS_RET_CRYPROV_ERR); } - CHKiRet(pData->cryprov.SetCnfParam(pData->cryprovData, lst)); + CHKiRet(pData->cryprov.SetCnfParam(pData->cryprovData, lst, CRYPROV_PARAMTYPE_REGULAR)); dbgprintf("loaded crypto provider %s, data instance at %p\n", szDrvrName, pData->cryprovData); diff --git a/tools/omfwd.c b/tools/omfwd.c index 129392d2..179e9614 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -4,7 +4,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2007-2012 Adiscon GmbH. + * Copyright 2007-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -39,6 +39,7 @@ #include <errno.h> #include <ctype.h> #include <unistd.h> +#include <stdint.h> #ifdef USE_NETZIP #include <zlib.h> #endif @@ -97,6 +98,13 @@ typedef struct _instanceData { TCPFRAMINGMODE tcp_framing; int bResendLastOnRecon; /* should the last message be re-sent on a successful reconnect? */ tcpclt_t *pTCPClt; /* our tcpclt object */ +# define COMPRESS_NEVER 0 +# define COMPRESS_SINGLE_MSG 1 /* old, single-message compression */ + /* all other settings are for stream-compression */ +# define COMPRESS_STREAM_ALWAYS 2 + uint8_t compressionMode; + sbool bzInitDone; /* did we do an init of zstrm already? */ + z_stream zstrm; /* zip stream to use for tcp compression */ uchar sndBuf[16*1024]; /* this is intensionally fixed -- see no good reason to make configurable */ unsigned offsSndBuf; /* next free spot in send buffer */ } instanceData; @@ -132,6 +140,7 @@ static struct cnfparamdescr actpdescr[] = { { "protocol", eCmdHdlrGetWord, 0 }, { "tcp_framing", eCmdHdlrGetWord, 0 }, { "ziplevel", eCmdHdlrInt, 0 }, + { "compression.mode", eCmdHdlrGetWord, 0 }, { "rebindinterval", eCmdHdlrInt, 0 }, { "streamdriver", eCmdHdlrGetWord, 0 }, { "streamdrivermode", eCmdHdlrInt, 0 }, @@ -169,6 +178,7 @@ ENDinitConfVars static rsRetVal doTryResume(instanceData *pData); +static rsRetVal doZipFinish(instanceData *pData); /* this function gets the default template. It coordinates action between * old-style and new-style configuration parts. @@ -240,6 +250,7 @@ static inline void DestructTCPInstanceData(instanceData *pData) { assert(pData != NULL); + doZipFinish(pData); if(pData->pNetstrm != NULL) netstrm.Destruct(&pData->pNetstrm); if(pData->pNS != NULL) @@ -423,14 +434,8 @@ finalize_it: /* CODE FOR SENDING TCP MESSAGES */ - -/* Send a buffer via TCP. Usually, this is used to send the current - * send buffer, but if a message is larger than the buffer, we need to - * have the capability to send the message buffer directly. - * rgerhards, 2011-04-04 - */ static rsRetVal -TCPSendBuf(instanceData *pData, uchar *buf, unsigned len) +TCPSendBufUncompressed(instanceData *pData, uchar *buf, unsigned len) { DEFiRet; unsigned alreadySent; @@ -438,6 +443,7 @@ TCPSendBuf(instanceData *pData, uchar *buf, unsigned len) alreadySent = 0; CHKiRet(netstrm.CheckConnection(pData->pNetstrm)); /* hack for plain tcp syslog - see ptcp driver for details */ + while(alreadySent != len) { lenSend = len - alreadySent; CHKiRet(netstrm.Send(pData->pNetstrm, buf+alreadySent, &lenSend)); @@ -455,6 +461,99 @@ finalize_it: RETiRet; } +static rsRetVal +TCPSendBufCompressed(instanceData *pData, uchar *buf, unsigned len) +{ + int zRet; /* zlib return state */ + unsigned outavail; + uchar zipBuf[32*1024]; + DEFiRet; + + if(!pData->bzInitDone) { + /* allocate deflate state */ + pData->zstrm.zalloc = Z_NULL; + pData->zstrm.zfree = Z_NULL; + pData->zstrm.opaque = Z_NULL; + /* see note in file header for the params we use with deflateInit2() */ + zRet = deflateInit(&pData->zstrm, 9); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateInit()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + pData->bzInitDone = RSTRUE; + } + + /* now doing the compression */ + pData->zstrm.next_in = (Bytef*) buf; + pData->zstrm.avail_in = len; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("omfwd: in deflate() loop, avail_in %d, total_in %ld\n", pData->zstrm.avail_in, pData->zstrm.total_in); + pData->zstrm.avail_out = sizeof(zipBuf); + pData->zstrm.next_out = zipBuf; + zRet = deflate(&pData->zstrm, Z_NO_FLUSH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pData->zstrm.avail_out); + outavail = sizeof(zipBuf) - pData->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(TCPSendBufUncompressed(pData, zipBuf, outavail)); + } + } while (pData->zstrm.avail_out == 0); + +finalize_it: + RETiRet; +} + +static rsRetVal +TCPSendBuf(instanceData *pData, uchar *buf, unsigned len) +{ + DEFiRet; + if(pData->compressionMode >= COMPRESS_STREAM_ALWAYS) + iRet = TCPSendBufCompressed(pData, buf, len); + else + iRet = TCPSendBufUncompressed(pData, buf, len); + RETiRet; +} + +/* finish zlib buffer, to be called before closing the ZIP file (if + * running in stream mode). + */ +static rsRetVal +doZipFinish(instanceData *pData) +{ + int zRet; /* zlib return state */ + DEFiRet; + unsigned outavail; + uchar zipBuf[32*1024]; + + if(!pData->bzInitDone) + goto done; + +// TODO: can we get this into a single common function? +dbgprintf("DDDD: in doZipFinish()\n"); + pData->zstrm.avail_in = 0; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pData->zstrm.avail_in, pData->zstrm.total_in); + pData->zstrm.avail_out = sizeof(zipBuf); + pData->zstrm.next_out = zipBuf; + zRet = deflate(&pData->zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pData->zstrm.avail_out); + outavail = sizeof(zipBuf) - pData->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(TCPSendBufUncompressed(pData, zipBuf, outavail)); + } + } while (pData->zstrm.avail_out == 0); + +finalize_it: + zRet = deflateEnd(&pData->zstrm); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); + } + + pData->bzInitDone = 0; +done: RETiRet; +} + /* Add frame to send buffer (or send, if requried) */ @@ -636,11 +735,10 @@ CODESTARTdoAction * hard-coded but this may be changed to a config parameter. * rgerhards, 2006-11-30 */ - if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { + if(pData->compressionMode == COMPRESS_SINGLE_MSG && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { uLongf destLen = iMaxLine + iMaxLine/100 +12; /* recommended value from zlib doc */ uLong srcLen = l; int ret; - /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ CHKmalloc(out = (Bytef*) MALLOC(destLen)); out[0] = 'z'; out[1] = '\0'; @@ -756,14 +854,17 @@ setInstParamDefaults(instanceData *pData) pData->iRebindInterval = 0; pData->bResendLastOnRecon = 0; pData->pPermPeers = NULL; - pData->compressionLevel = 0; + pData->compressionLevel = 9; + pData->compressionMode = COMPRESS_NEVER; } BEGINnewActInst struct cnfparamvals *pvals; uchar *tplToUse; + char *cstr; int i; rsRetVal localRet; + int complevel = -1; CODESTARTnewActInst DBGPRINTF("newActInst (omfwd)\n"); @@ -860,9 +961,10 @@ CODESTARTnewActInst free(str); } else if(!strcmp(actpblk.descr[i].name, "ziplevel")) { # ifdef USE_NETZIP - int complevel = pvals[i].val.d.n; + complevel = pvals[i].val.d.n; if(complevel >= 0 && complevel <= 10) { pData->compressionLevel = complevel; + pData->compressionMode = COMPRESS_SINGLE_MSG; } else { errmsg.LogError(0, NO_ERRCODE, "Invalid ziplevel %d specified in " "forwardig action - NOT turning on compression.", @@ -876,11 +978,37 @@ CODESTARTnewActInst pData->bResendLastOnRecon = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "template")) { pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "compression.mode")) { + cstr = es_str2cstr(pvals[i].val.d.estr, NULL); + if(!strcasecmp(cstr, "stream:always")) { + pData->compressionMode = COMPRESS_STREAM_ALWAYS; + } else if(!strcasecmp(cstr, "none")) { + pData->compressionMode = COMPRESS_NEVER; + } else if(!strcasecmp(cstr, "single")) { + pData->compressionMode = COMPRESS_SINGLE_MSG; + } else { + errmsg.LogError(0, RS_RET_PARAM_ERROR, "omfwd: invalid value for 'compression.mode' " + "parameter (given is '%s')", cstr); + free(cstr); + ABORT_FINALIZE(RS_RET_PARAM_ERROR); + } + free(cstr); } else { DBGPRINTF("omfwd: program error, non-handled " "param '%s'\n", actpblk.descr[i].name); } } + + if(complevel != -1) { + pData->compressionLevel = complevel; + if(pData->compressionMode == COMPRESS_NEVER) { + /* to keep compatible with pre-7.3.11, only setting the + * compresion level means old-style single-message mode. + */ + pData->compressionMode = COMPRESS_SINGLE_MSG; + } + } + CODE_STD_STRING_REQUESTnewActInst(1) tplToUse = ustrdup((pData->tplName == NULL) ? getDfltTpl() : pData->tplName); @@ -948,6 +1076,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) iLevel = *p - '0'; ++p; /* eat */ pData->compressionLevel = iLevel; + pData->compressionMode = COMPRESS_SINGLE_MSG; } else { errmsg.LogError(0, NO_ERRCODE, "Invalid compression level '%c' specified in " "forwardig action - NOT turning on compression.", @@ -1025,7 +1154,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) while(*p && *p != ';' && *p != '#' && !isspace((int) *p)) ++p; /*JUST SKIP*/ - /* TODO: make this if go away! */ if(*p == ';' || *p == '#' || isspace(*p)) { uchar cTmp = *p; *p = '\0'; /* trick to obtain hostname (later)! */ diff --git a/tools/rsyslogd.8 b/tools/rsyslogd.8 index 620006f2..ac732b88 100644 --- a/tools/rsyslogd.8 +++ b/tools/rsyslogd.8 @@ -191,6 +191,10 @@ is specified and the host logging resolves to satu.infodrom.north.de no domain would be cut, you will have to specify two domains like: .BR "\-s north.de:infodrom.north.de" . .TP +.BI "\-S ip_address" "local client source IP" +rsyslogd uses ip_address as local client address while connecting +to remote logserver. Currently used by omrelp only and only with tcp. +.TP .BI "\-u " "userlevel" This is a "catch all" option for some very seldomly-used user settings. The "userlevel" variable selects multiple things. Add the specific values diff --git a/tools/syslogd.c b/tools/syslogd.c index a8a733d6..2f0f64c3 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1056,7 +1056,7 @@ finalize_it: * the time being (remember that we want to restructure config processing at large!). * rgerhards, 2009-10-27 */ -rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct cnfparamvals *queueParams) +rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct nvlst *lst) { struct queuefilenames_s *qfn; uchar *qfname = NULL; @@ -1072,7 +1072,7 @@ rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct cnfpara /* name our main queue object (it's not fatal if it fails...) */ obj.SetName((obj_t*) (*ppQueue), pszQueueName); - if(queueParams == NULL) { /* use legacy parameters? */ + if(lst == NULL) { /* use legacy parameters? */ /* ... set some properties ... */ # define setQPROP(func, directive, data) \ CHKiRet_Hdlr(func(*ppQueue, data)) { \ @@ -1129,7 +1129,7 @@ rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct cnfpara # undef setQPROPstr } else { /* use new style config! */ qqueueSetDefaultsRulesetQueue(*ppQueue); - qqueueApplyCnfParam(*ppQueue, queueParams); + qqueueApplyCnfParam(*ppQueue, lst); } /* ... and finally start the queue! */ @@ -1772,7 +1772,7 @@ int realMain(int argc, char **argv) * of other options, we do this during the inital option processing. * rgerhards, 2008-04-04 */ - while((ch = getopt(argc, argv, "46a:Ac:dDef:g:hi:l:m:M:nN:op:qQr::s:t:T:u:vwx")) != EOF) { + while((ch = getopt(argc, argv, "46a:Ac:dDef:g:hi:l:m:M:nN:op:qQr::s:S:t:T:u:vwx")) != EOF) { switch((char)ch) { case '4': case '6': @@ -1790,6 +1790,7 @@ int realMain(int argc, char **argv) case 'q': /* add hostname if DNS resolving has failed */ case 'Q': /* dont resolve hostnames in ACL to IPs */ case 's': + case 'S': /* Source IP for local client to be used on multihomed host */ case 'T': /* chroot on startup (primarily for testing) */ case 'u': /* misc user settings */ case 'w': /* disable disallowed host warnings */ @@ -1881,6 +1882,13 @@ int realMain(int argc, char **argv) case 'a': fprintf(stderr, "rsyslogd: error -a is no longer supported, use module imuxsock instead"); break; + case 'S': /* Source IP for local client to be used on multihomed host */ + if(glbl.GetSourceIPofLocalClient() != NULL) { + fprintf (stderr, "rsyslogd: Only one -S argument allowed, the first one is taken.\n"); + } else { + glbl.SetSourceIPofLocalClient((uchar*)arg); + } + break; case 'f': /* configuration file */ ConfFile = (uchar*) arg; break; |