diff options
Diffstat (limited to 'doc')
57 files changed, 2505 insertions, 448 deletions
diff --git a/doc/Makefile.am b/doc/Makefile.am index e8cdba55..e1757644 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -34,11 +34,15 @@ html_files = \ ompipe.html \ omfwd.html \ omfile.html \ + omjournal.html \ + imjournal.html \ + mmanon.html \ omusrmsg.html \ omstdout.html \ omudpspoof.html \ omruleset.html \ omsnmp.html \ + sigprov_gt.html \ ommysql.html \ omoracle.html \ omlibdbi.html \ @@ -70,6 +74,7 @@ html_files = \ tls_cert_client.html \ tls_cert_scenario.html \ rainerscript.html \ + lookup_tables.html \ rscript_abnf.html \ rsconf1_actionexeconlywhenpreviousissuspended.html \ rsconf1_actionresumeinterval.html \ @@ -115,6 +120,7 @@ html_files = \ gssapi.html \ licensing.html \ mmnormalize.html \ + mmjsonparse.html \ ommail.html \ omuxsock.html \ omrelp.html \ diff --git a/doc/build_from_repo.html b/doc/build_from_repo.html index a06863e1..693704a8 100644 --- a/doc/build_from_repo.html +++ b/doc/build_from_repo.html @@ -8,7 +8,7 @@ distribution tarball to generate it. But there may be situations where it is des to build directly from the source repository. This is useful for people who would like to participate in development or who would like to use the latest, not-yet-released code. The later may especially be the case if you are asked to try out an experimental version. -<p>Building from the repsitory is not much different than building from the source +<p>Building from the repository is not much different than building from the source tarball, but some files are missing because they are output files and thus do not belong into the repository. <h2>Obtaining the Source</h2> diff --git a/doc/confsamples/normalization.conf b/doc/confsamples/normalization.conf new file mode 100644 index 00000000..7cfd92ef --- /dev/null +++ b/doc/confsamples/normalization.conf @@ -0,0 +1,187 @@ +# this is a config sample for log normalization, but can +# be used as a more complex general sample. +# It is based on a plain standard rsyslog.conf for Red Hat systems. +# +# NOTE: Absolute path names for modules are used in this config +# so that we can run a different rsyslog version alongside the +# regular system-installed rsyslogd. Remove these path names +# for production environment. + +#### MODULES #### + +# we do not run imuxsock as we don't want to mess with the main system logger +#module(load="/home/rger/proj/rsyslog/plugins/imuxsock/.libs/imuxsock") # provides support for local system logging (e.g. via logger command) +#module(load="imklog") # provides kernel logging support (previously done by rklogd) +module(load="/home/rger/proj/rsyslog/plugins/imudp/.libs/imudp") # Provides UDP syslog reception +module(load="/home/rger/proj/rsyslog/plugins/imtcp/.libs/imtcp") +module(load="/home/rger/proj/rsyslog/plugins/mmjsonparse/.libs/mmjsonparse") +module(load="/home/rger/proj/rsyslog/plugins/mmnormalize/.libs/mmnormalize") + +/* We assume to have all TCP logging (for simplicity) + * Note that we use different ports to point different sources + * to the right rule sets for normalization. While there are + * other methods (e.g. based on tag or source), using multiple + * ports is both the easiest as well as the fastest. + */ +input(type="imtcp" port="13514" Ruleset="WindowsRsyslog") +input(type="imtcp" port="13515" Ruleset="LinuxPlainText") +input(type="imtcp" port="13516" Ruleset="WindowsSnare") + +#debug: +action(type="omfile" file="/home/rger/proj/rsyslog/logfile") + +/* This ruleset handles structured logging. + * It is the only one ever called for remote machines + * but executed in addition to the standard action for + * the local machine. The ultimate goal is to forward + * to some Vendor's analysis tool (which digests a + * structured log format, here we use Lumberjack). + */ +template(name="lumberjack" type="string" string="%$!all-json%\n") + + +/* the rsyslog Windows Agent uses native Lumberjack format + * (better said: is configured to use it) + */ +ruleset(name="WindowsRsyslog") { + action(type="mmjsonparse") + if $parsesuccess == "OK" then { + if $!id == 4634 then + set $!usr!type = "logoff"; + else if $!id == 4624 then + set $!usr!type = "logon"; + set $!usr!rcvdfrom = $!source; + set $!usr!rcvdat = $timereported; + set $!usr!user = $!TargetDomainName & "\\" & $!TargetUserName; + call outwriter + } +} + +/* This handles clumsy snare format. Note that "#011" are + * the escape sequences for tab chars used by snare. + */ +ruleset(name="WindowsSnare") { + set $!usr!type = field($rawmsg, "#011", 6); + if $!usr!type == 4634 then { + set $!usr!type = "logoff"; + set $!doProces = 1; + } else if $!usr!type == 4624 then { + set $!usr!type = "logon"; + set $!doProces = 1; + } else + set $!doProces = 0; + if $!doProces == 1 then { + set $!usr!rcvdfrom = field($rawmsg, 32, 4); + set $!usr!rcvdat = field($rawmsg, "#011", 5); + /* we need to fix up the snare date */ + set $!usr!rcvdat = field($!usr!rcvdat, 32, 2) & " " & + field($!usr!rcvdat, 32, 3) & " " & + field($!usr!rcvdat, 32, 4); + set $!usr!user = field($rawmsg, "#011", 8); + call outwriter + } +} + +/* plain Linux log messages (here: ssh and sudo) need to be + * parsed - we use mmnormalize for fast and efficient parsing + * here. + */ +ruleset(name="LinuxPlainText") { + action(type="mmnormalize" + rulebase="/home/rger/proj/rsyslog/linux.rb" userawmsg="on") + if $parsesuccess == "OK" and $!user != "" then { + if $!type == "opened" then + set $!usr!type = "logon"; + else if $!type == "closed" then + set $!usr!type = "logoff"; + set $!usr!rcvdfrom = $!rcvdfrom; + set $!usr!rcvdat = $!rcvdat; + set $!usr!user = $!user; + call outwriter + } +} + +/* with CSV, we the reader must receive information on the + * field names via some other method (e.g. tool configuration, + * prepending of a header to the written CSV-file). All of + * this is highly dependant on the actual CSV dialect needed. + * Below, we cover the basics. + */ +template(name="csv" type="list") { + property(name="$!usr!rcvdat" format="csv") + constant(value=",") + property(name="$!usr!rcvdfrom" format="csv") + constant(value=",") + property(name="$!usr!user" format="csv") + constant(value=",") + property(name="$!usr!type" format="csv") + constant(value="\n") +} + +/* template for Lumberjack-style logging. Note that the extra + * LF at the end is just for wrinting it to file - it MUST NOT + * be included for messages intended to be sent to a remote system. + * For the latter use case, the syslog header must also be prepended, + * something we have also not done for simplicity (as we write to files). + * Note that we use a JSON-shortcut: If a tree name is specified, JSON + * for its whole subtree is generated. Thus, we only need to specify the + * $!usr top node to get everytihing we need. + */ +template(name="cee" type="string" string="@cee: %$!usr%\n") + + +/* this ruleset simulates forwarding to the final destination */ +ruleset(name="outwriter"){ + action(type="omfile" + file="/home/rger/proj/rsyslog/logfile.csv" template="csv") + action(type="omfile" + file="/home/rger/proj/rsyslog/logfile.cee" template="cee") +} + + +/* below is just the usual "uninteresting" stuff... + * Note that this goes into the default rule set. So + * local logging is handled "as usual" without the need + * for any extra effort. + */ + + +#### GLOBAL DIRECTIVES #### + +# Use default timestamp format +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# Include all config files in /etc/rsyslog.d/ +# commented out not to interfere with the system rsyslogd +# (just for this test configuration!) +#$IncludeConfig /etc/rsyslog.d/*.conf + + +#### RULES #### + +# Log all kernel messages to the console. +# Logging much else clutters up the screen. +#kern.* /dev/console + +# Log anything (except mail) of level info or higher. +# Don't log private authentication messages! +*.info;mail.none;authpriv.none;cron.none /var/log/messages + +# The authpriv file has restricted access. +authpriv.* /var/log/secure + +# Log all the mail messages in one place. +mail.* /var/log/maillog + + +# Log cron stuff +cron.* /var/log/cron + +# Everybody gets emergency messages +*.emerg :omusrmsg:* + +# Save news errors of level crit and higher in a special file. +uucp,news.crit /var/log/spooler + +# Save boot messages also to boot.log +local7.* /var/log/boot.log diff --git a/doc/cryprov_gcry.html b/doc/cryprov_gcry.html new file mode 100644 index 00000000..2568add9 --- /dev/null +++ b/doc/cryprov_gcry.html @@ -0,0 +1,121 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>libgcryt Log Crypto Provider (gcry)</title> +</head> + +<body> +<a href="rsyslog_conf_modules.html">back to rsyslog module overview</a> + +<h1>libgcrypt Log Crypto Provider (gcry)</h1> +<p><b>Signature Provider Name: gt</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Supported Since: </b>since 7.3.10 +<p><b>Description</b>:</p> +<p>Provides encryption support to rsyslog. +</p> + +<p><b>Configuration Parameters</b>:</p> +<p>Crypto providers are loaded by omfile, when the +provider is selected in its "cry.providerName" parameter. +Parameters for the provider are given in the omfile action instance +line. +<p>This provider creates an encryption information file with the same base name but +the extension ".encinfo" for each log file (both for fixed-name files +as well as dynafiles). Both files together form a set. So you need to +archive both in order to prove integrity. +<ul> +<li><b>cry.algo</b> <Encryption Algorithm><br> +The algorithm (cipher) to be used for encryption. +The default algorithm is "AES128". +<br>Currently, the following Algorithms are supported: + <ul> + <li>3DES + <li>CAST5 + <li>BLOWFISH + <li>AES128 + <li>AES192 + <li>AES256 + <li>TWOFISH + <li>TWOFISH128 + <li>ARCFOUR + <li>DES + <li>SERPENT128 + <li>SERPENT192 + <li>SERPENT256 + <li>RFC2268_40 + <li>SEED + <li>CAMELLIA128 + <li>CAMELLIA192 + <li>CAMELLIA256 + </ul> + <br> + The actual availability of an algorithms depends on which ones + are compiled into libgcrypt. Note that some versions of libgcrypt + simply abort the process (rsyslogd in this case!) if a supported + algorithm is select but not available due to libgcrypt build + settings. There is nothing rsyslog can do against this. So in + order to avoid production downtime, always check carefully when + you change the algorithm. +</li> +<li><b>cry.mode</b> <Algorithm Mode><br> +The encryption mode to be used. Default ist Cipher Block Chaining (CBC). +Note that not all encryption modes can be used together with all +algorithms. +<br>Currently, the following modes are supported: + <ul> + <li>ECB + <li>CFB + <li>CBC + <li>STREAM + <li>OFB + <li>CTR + <li>AESWRAP + </ul> +<li><b>cry.key</b> <encryption key><br> + TESTING AID, NOT FOR PRODUCTION USE. This uses the KEY specified + inside rsyslog.conf. This is the actual key, and as such this mode + is highly insecure. However, it can be useful for intial testing + steps. This option may be removed in the future. +</li> +<li><b>cry.keyfile</b> <filename><br> + Reads the key from the specified file. The file must contain the key, only, + no headers or other meta information. Keyfiles can be generated via the + rscrytool utility. +</li> +<li><b>cry.keyprogram</b> <path to program><br> + If given, the key is provided by a so-called "key program". This program + is executed and must return the key to (as well as some meta information) + via stdout. The core idea of key programs is that using this interface the + user can implement as complex (and secure) method to obtain keys as + desired, all without the need to make modifications to rsyslog. +</li> +</ul> +<b>Caveats/Known Bugs:</b> +<ul> +<li>currently none known +</li> +</ul> +<p><b>Samples:</b></p> +<p>This encrypts a log file. Default parameters are used, they key is +provided via a keyfile. +</p> +<textarea rows="3" cols="60"> +action(type="omfile" file="/var/log/somelog" + cry.provider="gcry" keyfile="/secured/path/to/keyfile") +</textarea> +Note that the keyfile can be generated via the rscrytool utility (see its +documentation for how to actually do that). + + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2013 by +<a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/debug.html b/doc/debug.html index 6aeb7975..537cd6b4 100644 --- a/doc/debug.html +++ b/doc/debug.html @@ -27,7 +27,7 @@ be replaced by something else.</p> <p>There are two environment variables that set several debug settings: <ul> <li>The "RSYSLOG_DEBUGLOG" (sample: RSYSLOG_DEBUGLOG="/path/to/debuglog/") -writes (allmost) +writes (almost) all debug message to the specified log file in addition to stdout. Some system messages (e.g. segfault or abort message) are not written to the file as we can not capture them. @@ -49,8 +49,14 @@ FileTrace=vm.c FileTrace=expr.c"</li> <li><b>Debug</b> - if present, turns on the debug system and enables debug output <li><b>DebugOnDemand</b> - if present, turns on the debug system but does not enable debug output itself. You need to send SIGUSR1 to turn it on when desired. +<li><b>OutputTidToStderr</b> - if present, makes rsyslog output information about +the thread id (tid) of newly create processesto stderr. Note that not necessarily +all new threads are reported (depends on the code, e.g. of plugins). This is +only available under Linux. This usually does NOT work when privileges have +been dropped (that's not a bug, but the way it is). <li><b>help</b> - display a very short list of commands - hopefully a life saver if you can't access the documentation...</li> </ul> +<p>Individual options are separated by spaces.</p> </ul> <h3>Why Environment Variables?</h3> <p>You may ask why we use environment variables for debug-system parameters and not @@ -70,6 +76,26 @@ rsyslog core, we get a number of data structures wrong. <p>For these reasons, we utilize environment variables to initialize and configure the debugging system. We understand this may be somewhat painful, but now you know there are at least some good reasons for doing so. +<p>HOWEVER, if you have a too hard time to set debug instructions using the environment +variables, there is a cure, described in the next paragraph. + +<h2>Enabling Debug via rsyslog.conf</h2> +<p>As described in the previous paragraph, enabling debug via rsyslog.conf +may not be perfect for some debugging needs, but basic debug output will work - and +that is what most often is requried. There are limited options available, but these +cover the most important use cases. +<p>Debug processing is done via legacy config statements. There currently +is no plan to move these over to the v6+ config system. Availabe settings are +<ul> +<li>$DebugFile <filename> - sets the debug file name +<li>$DebugLevel <0|1|2> - sets the respective debug level, where +0 means debug off, 1 is debug on demand activated (but debug mode off) +and 2 is full debug mode. +</ul> +<p>Note that in theory it is forbidden to specify these parameters more +than once. However, we do not enforce that and if it happens results +are undefined. + <h2>Getting debug information from a running Instance</h2> <p>It is possible to obtain debugging information from a running instance, but this requires some setup. We assume that the instance runs in the background, so debug output to @@ -107,7 +133,7 @@ turned on. threads and their calling stack by sending SIGUSR2. However, the usefulness of that information is very much depending on rsyslog compile-time settings, must importantly the --enable-rtinst configure flag. Note that activating this option causes additional overhead -and slows down rsyslgod considerable. So if you do that, you need to check if it is +and slows down rsyslogd considerable. So if you do that, you need to check if it is capable to handle the workload. Also, threading behavior is modified by the runtime instrumentation. <p>Sending SIGUSR2 writes new process state information to the log file each time @@ -117,13 +143,13 @@ some diagnostic information on the current processing state. In that case, turni on the mutex debugging options (see above) is probably useful. <h2>Interpreting the Logs</h2> <p>Debug logs are primarily meant for rsyslog developers. But they may still provide valuable -information to users. Just be warned that logs sometimes contains informaton the looks like +information to users. Just be warned that logs sometimes contains information the looks like an error, but actually is none. We put a lot of extra information into the logs, and there are some cases where it is OK for an error to happen, we just wanted to record it inside the log. The code handles many cases automatically. So, in short, the log may not make sense to you, but it (hopefully) makes sense to a developer. Note that we developers often need many lines of the log file, it is relatively rare that a problem can be diagnosed by -looking at just a couple of (hundered) log records. +looking at just a couple of (hundred) log records. <h2>Security Risks</h2> <p>The debug log will reveal potentially sensible information, including user accounts and passwords, to anyone able to read the log file. As such, it is recommended to properly @@ -133,12 +159,12 @@ attack or try to hide some information from the log file. As such, it is suggest enable DebugOnDemand mode only for a reason. Note that when no debug mode is enabled, SIGUSR1 and SIGUSR2 are completely ignored. <p>When running in any of the debug modes (including on demand mode), an interactive -instance of rsyslogd can be aborted by pressing ctl-c. +instance of rsyslogd can be aborted by pressing ctrl-c. <p> <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body> diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html index b33b67f9..802de804 100644 --- a/doc/dev_oplugins.html +++ b/doc/dev_oplugins.html @@ -15,7 +15,7 @@ and pointers than to have nothing. <p>The best to get started with rsyslog plugin development is by looking at existing plugins. All that start with "om" are <b>o</b>utput <b>m</b>odules. That means they are primarily thought of being message sinks. In theory, however, output -plugins may aggergate other functionality, too. Nobody has taken this route so far +plugins may aggregate other functionality, too. Nobody has taken this route so far so if you would like to do that, it is highly suggested to post your plan on the rsyslog mailing list, first (so that we can offer advise). <p>The rsyslog distribution tarball contains two plugins that are extremely well @@ -29,7 +29,7 @@ plugins. It is bare of real functionality but has ample comments. Even if you de to start from another plugin (or even from scratch), be sure to read omtemplate source and comments first. The omstdout is primarily a testing aide, but offers support for the two different parameter-passing conventions plugins can use (plus the way to -differentiate between the two). It also is not bare of functionaly, only mostly +differentiate between the two). It also is not bare of functionality, only mostly bare of it ;). But you can actually execute it and play with it. <p>In any case, you should also read the comments in ./runtime/module-template.h. Output plugins are build based on a large set of code-generating macros. These @@ -65,7 +65,7 @@ that shares a single instanceData structure. <p>So as long as you do not mess around with global data, you do not need to think about multithreading (and can apply a purely sequential programming methodology). -<p>Please note that duringt the configuraton parsing stage of execution, access to +<p>Please note that during the configuration parsing stage of execution, access to global variables for the configuration system is safe. In that stage, the core will only call sequentially into the plugin. <h3>Getting Message Data</h3> @@ -90,7 +90,7 @@ get it into the core (so far, we could accept all such suggestions - no promise, request access to the template components. The typical use case seems to be databases, where you would like to access properties via specific fields. With that mode, you receive a char ** array, where each array element points to one field from the template (from -left to right). Fields start at arrray index 0 and a NULL pointer means you have +left to right). Fields start at array index 0 and a NULL pointer means you have reached the end of the array (the typical Unix "poor man's linked list in an array" design). Note, however, that each of the individual components is a string. It is not a date stamp, number or whatever, but a string. This is because rsyslog processes @@ -153,19 +153,19 @@ for example in MongoDB or ElasticSearch. a single-message interface was supported. <p>With the <b>single message</b> plugin interface, each message is passed via a separate call to the plugin. Most importantly, the rsyslog engine assumes that each call to the plugin is a complete transaction -and as such assumes that messages be properly commited after the plugin returns to the engine. +and as such assumes that messages be properly committed after the plugin returns to the engine. <p>With the <b>batching</b> interface, rsyslog employs something along the line of "transactions". Obviously, the rsyslog core can not make non-transactional outputs to be fully transactional. But what it can is support that the output tells the core which -messages have been commited by the output and which not yet. The core can than take care -of those uncommited messages when problems occur. For example, if a plugin has received -50 messages but not yet told the core that it commited them, and then returns an error state, the +messages have been committed by the output and which not yet. The core can than take care +of those uncommitted messages when problems occur. For example, if a plugin has received +50 messages but not yet told the core that it committed them, and then returns an error state, the core assumes that all these 50 messages were <b>not</b> written to the output. The core then -requeues all 50 messages and does the usual retry processing. Once the output plugin tells the +re-queues all 50 messages and does the usual retry processing. Once the output plugin tells the core that it is ready again to accept messages, the rsyslog core will provide it with these 50 -not yet commited messages again (actually, at this point, the rsyslog core no longer knows that -it is re-submiting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 -messages were commited (before it failed), then only 10 would have been requeued and resubmitted. +not yet committed messages again (actually, at this point, the rsyslog core no longer knows that +it is re-submitting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 +messages were committed (before it failed), then only 10 would have been re-queued and resubmitted. <p>In order to provide an efficient implementation, there are some (mild) constraints in that transactional model: first of all, rsyslog itself specifies the ultimate transaction boundaries. That is, it tells the plugin when a transaction begins and when it must finish. The plugin @@ -176,7 +176,7 @@ transaction support. Note that batch sizes are variable within the range of 1 to maximum limit. Most importantly, that means that plugins may receive batches of single messages, so they are required to commit each message individually. If the plugin tries to be "smarter" than the rsyslog engine and does not commit messages in those cases (for example), the plugin -puts message stream integrity at risk: once rsyslog has notified the plugin of transacton end, +puts message stream integrity at risk: once rsyslog has notified the plugin of transaction end, it discards all messages as it considers them committed and save. If now something goes wrong, the rsyslog core does not try to recover lost messages (and keep in mind that "goes wrong" includes such uncontrollable things like connection loss to a database server). So it is @@ -191,8 +191,8 @@ This is also under evaluation and, once decided, the core will offer an interfac to preserve message stream integrity for properly-crafted plugins). <p>The second restriction is that if a plugin makes commits in between (what is perfectly legal) those commits must be in-order. So if a commit is made for message ten out of 50, -this means that messages one to nine are also commited. It would be possible to remove -this restriction, but we have decided to deliberately introduce it to simpify things. +this means that messages one to nine are also committed. It would be possible to remove +this restriction, but we have decided to deliberately introduce it to simplify things. <h3>Output Plugin Transaction Interface</h3> <p>In order to keep compatible with existing output plugins (and because it introduces no complexity), the transactional plugin interface is build on the traditional @@ -252,7 +252,7 @@ But they convey additional information about the commit status as follows: <table border="0"> <tr> <td valign="top"><i>RS_RET_OK</i></td> -<td>The record and all previous inside the batch has been commited. +<td>The record and all previous inside the batch has been committed. <i>Note:</i> this definition is what makes integrating plugins without the transaction being/end calls so easy - this is the traditional "success" return state and if every call returns it, there is no need for actually calling @@ -260,7 +260,7 @@ state and if every call returns it, there is no need for actually calling </tr> <tr> <td valign="top"><i>RS_RET_DEFER_COMMIT</i></td> -<td>The record has been processed, but is not yet commited. This is the +<td>The record has been processed, but is not yet committed. This is the expected state for transactional-aware plugins.</td> </tr> <tr> @@ -269,7 +269,7 @@ expected state for transactional-aware plugins.</td> current one not yet. This state is introduced to support sources that fill up buffers and commit once a buffer is completely filled. That may occur halfway in the next record, so it may be important to be able to tell the -engine the everything up to the previouos record is commited</td> +engine the everything up to the previous record is committed</td> </tr> </table> <p>Note that the typical <b>calling cycle</b> is <code>beginTransaction()</code>, @@ -290,7 +290,7 @@ exists. So we introduce it with that release. What the means is if a rsyslog cor not provide this query interface, it is a core that was build before batching support was available. So the absence of a query interface indicates that the transactional interface is not available. One might now be tempted the think there is no need to do -the actual check, but is is recommended to ask the rsyslog engine explicitely if +the actual check, but is is recommended to ask the rsyslog engine explicitly if the transactional interface is present and will be honored. This enables us to create versions in the future which have, for whatever reason we do not yet know, no support for this interface. diff --git a/doc/dev_queue.html b/doc/dev_queue.html index bf2af7f0..6d5fe73f 100644 --- a/doc/dev_queue.html +++ b/doc/dev_queue.html @@ -80,7 +80,7 @@ and terminating while waiting on the primary queue to fill. In practice, this is highly unlikely (but only for the main message queue) because rsyslog issues a startup message. HOWEVER, we can not rely on that, it would introduce a race. If the primary rsyslog thread (the one that issues the message) is scheduled very -late and there is a low inactivty timeout for queue workers, the queue worker +late and there is a low inactivity timeout for queue workers, the queue worker may terminate before the startup message is issued. And if the on-disk queue holds only a few messages, it may become empty before the DA worker is re-initiated again. So it is possible that the DA run mode termination criteria @@ -105,7 +105,7 @@ clean shutdown of the DA queue).</p> <p>One might think that it would be more natural for the DA queue to detect being idle and shut down itself. However, there are some issues associated with that. Most importantly, all queue worker threads need to be shut down during -queue destruction. Only after that has happend, final destruction steps can +queue destruction. Only after that has happened, final destruction steps can happen (else we would have a myriad of races). However, it is the DA queues worker thread that detects it is empty (empty queue detection always happens at the consumer side and must so). That would lead to the DA queue worker thread to @@ -115,7 +115,7 @@ destructed). Obviously, this does not work out (and I didn't even mention the other issues - so let's forget about it). As such, the thread that enqueues messages must destruct the queue - and that is the primary queue's DA worker thread.</p> -<p>There are some subleties due to thread synchronization and the fact that the +<p>There are some subtleties due to thread synchronization and the fact that the DA consumer may not be running (in a <b>case-2 startup</b>). So it is not trivial to reliably change the queue back from DA run mode to regular run mode. The priority is a clean switch. We accept the fact that there may be situations @@ -132,7 +132,7 @@ most probably even lead to worse performance under regular conditions).</p> </ol> <p>Case 2 is unlikely, but may happen (see info above on a case 2 startup).</p> <p><b>The DA worker may also not wait at all,</b> because it is actively -executing and shuffeling messages between the queues. In that case, however, the +executing and shuffling messages between the queues. In that case, however, the program flow passes both of the two wait conditions but simply does not wait.</p> <p><b>Finally, the DA worker may be inactive </b>(again, with a case-2 startup). In that case no work(er) at all is executed. Most importantly, without the DA @@ -247,4 +247,4 @@ no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at <a href="http://www.gnu.org/copyleft/fdl.html"> http://www.gnu.org/copyleft/fdl.html</a>.</p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/droppriv.html b/doc/droppriv.html index 7293e872..75773e64 100644 --- a/doc/droppriv.html +++ b/doc/droppriv.html @@ -18,7 +18,7 @@ rsyslogd needs to start up as root. user. That is probably the safest way of operations. However, if a startup as root is required, you can use the $PrivDropToGroup and $PrivDropToUser config directives to specify a group and/or user that rsyslogd should drop to after initialization. -Once this happend, the daemon runs without high privileges (depending, of +Once this happens, the daemon runs without high privileges (depending, of course, on the permissions of the user account you specified). <p>There is some additional information available in the <a href="http://wiki.rsyslog.com/index.php/Security#Dropping_Privileges">rsyslog wiki</a>. diff --git a/doc/features.html b/doc/features.html index 626ff65d..ecf6a014 100644 --- a/doc/features.html +++ b/doc/features.html @@ -42,7 +42,7 @@ into syslog messages (one per line)</li> <li>ability to configure backup syslog/database servers - if the primary fails, control is switched to a prioritized list of backups</li> <li>support for receiving messages via reliable <a href="http://www.monitorware.com/Common/en/glossary/rfc3195.php"> -RFC 3195</a> delivery (a bit clumpsy to build right now...)</li> +RFC 3195</a> delivery (a bit clumsy to build right now...)</li> <li>ability to generate file names and directories (log targets) dynamically, based on many different properties</li> <li>control of log output format, including ability to present @@ -95,7 +95,7 @@ via custom plugins</li> <li> an easy-to-write to plugin interface</li> <li> ability to send SNMP trap messages</li> <li> ability to filter out messages based on sequence of arrival</li> -<li>support for comma-seperated-values (CSV) output generation +<li>support for comma-separated-values (CSV) output generation (via the "csv" property replace option). The CSV format supported is that from RFC 4180.</li> <li>support for arbitrary complex boolean, string and diff --git a/doc/free_support.html b/doc/free_support.html index 182a82cd..a3a9a69a 100644 --- a/doc/free_support.html +++ b/doc/free_support.html @@ -18,7 +18,7 @@ reason is quite simple: If I do personal support, you gain some advantage withou contributing something back. Think about it: if you ask your question on the public forum or mailing list, other with the same problem can you and, most importantly, even years later find your post (and the answer) and get the problem solved. So by -solving your issue in public, you help create a great community ressource and also +solving your issue in public, you help create a great community resource and also help your fellow users finding solutions quicker. In the long term, this also contributes to improved code because the more questions users can find solutions to themselves, the fewer I need to look at. diff --git a/doc/history.html b/doc/history.html index a06aaf5d..a071461e 100644 --- a/doc/history.html +++ b/doc/history.html @@ -117,8 +117,26 @@ the need to have complex expression support, which was also the first use case. On February, 28th rsyslog 3.12.0 was released, the first version to contain expression support. This also meant that rsyslog from that date on supported all syslog-ng major features, but had a -number of major features exlusive to it. With 3.12.0, I consider -rsyslog fully superior to syslog-ng (except for platform support).</p><p>Be sure to visit Rainer's <a href="http://rgerhards.blogspot.com/">syslog blog</a> +number of major features exclusive to it. With 3.12.0, I consider +rsyslog fully superior to syslog-ng (except for platform support).</p> + +<p>Following the Fedora Developer's conference in Brno <b>2012</b>, rsyslog +got very serious on implementing <b>structured logging</b> in +project Lumberjack (CEE) style. Project Lumberjack was a much broader +effort and brought closer collaboration with the syslog-ng folks, which +helped to maintain and improve interoperability. In the +<b>late winter/spring/summer 2012</b> timeframe numerous engine enhancements +were made and plugins written (among them the first "official" interfaces +to the Linux audit subsystem). At the end of the year, this culminated in the +rsyslog 7, which not only implemented Lumberjack but also was the first one +to support full condition nesting in rsyslog.conf (and a ton of other features as +well). + +<p>In <b>spring 2013</b> major new security features were engineered, +namely anonymization support, as well as log file signing and +encryption capabilities. + +<p>Be sure to visit Rainer's <a href="http://rgerhards.blogspot.com/">syslog blog</a> to get some more insight into the development and futures of rsyslog and syslog in general. Don't be shy to post to either the blog or the <a href="http://www.rsyslog.com/PNphpBB2.phtml">rsyslog forums</a>.</p> @@ -126,4 +144,4 @@ Don't be shy to post to either the blog or the <ul> <li><a href="http://www.rsyslog.com/Topic4.phtml">the rsyslog change log</a></li> </ul> -</body></html>
\ No newline at end of file +</body></html> diff --git a/doc/im3195.html b/doc/im3195.html index aad9f3d1..317ab840 100644 --- a/doc/im3195.html +++ b/doc/im3195.html @@ -25,7 +25,7 @@ The port on which imklog listens for RFC 3195 messages. The default port is 601 to this input module, but we have NOT conducted any testing. Also, the module does not yet properly handle the recovery case. If someone intends to put this module into production, good testing should be -cunducted. It also is a good idea to notify the rsyslog project that you intend to use +conducted. It also is a good idea to notify the rsyslog project that you intend to use it in production. In this case, we'll probably give the module another cleanup. We don't do this now because so far it looks just like a big waste of time. diff --git a/doc/imfile.html b/doc/imfile.html index 0997e382..942fe531 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -14,7 +14,7 @@ a syslog message. A standard text file is a file consisting of printable characters with lines being delimited by LF.</p> <p>The file is read line-by-line and any line read is passed to -rsyslog's rule engine. The rule engine applies filter conditons and +rsyslog's rule engine. The rule engine applies filter conditions and selects which actions needs to be carried out. Empty lines are <b>not</b> processed, as they would result in empty syslog records. They are simply ignored.</p> @@ -49,9 +49,9 @@ releases of imfile may support per-file polling intervals, but currently this is not the case. If multiple PollingInterval statements are present in rsyslog.conf, only the last one is used.<br> A short poll interval provides more rapid message forwarding, but -requires more system ressources. While it is possible, we stongly +requires more system resources. While it is possible, we stongly recommend not to set the polling interval to 0 seconds. That will make -rsyslogd become a CPU hog, taking up considerable ressources. It is +rsyslogd become a CPU hog, taking up considerable resources. It is supported, however, for the few very unusual situations where this level may be needed. Even if you need quick response, 1 seconds should be well enough. Please note that imfile keeps reading files as long as @@ -61,15 +61,15 @@ nothing is left to be processed.</li> <p><b>Action Directives</b></p> <ul> -<li><strong>File /path/to/file</strong><br> +<li><strong>(required) File /path/to/file</strong><br> The file being monitored. So far, this must be an absolute name (no macros or templates)</li> -<li><span style="font-weight: bold;">Tag +<li><span style="font-weight: bold;">(required) Tag tag:</span><br> The tag to be used for messages that originate from this file. If you would like to see the colon after the tag, you need to specify it here (as shown above).</li> -<li><span style="font-weight: bold;">StateFile +<li><span style="font-weight: bold;">(required) StateFile <name-of-state-file></span><br> Rsyslog must keep track of which parts of the to be monitored file it already processed. This is done in the state file. This file always is @@ -77,7 +77,9 @@ created in the rsyslog working directory (configurable via $WorkDirectory). Be careful to use unique names for different files being monitored. If there are duplicates, all sorts of "interesting" things may happen. Rsyslog currently does not check if a name is -specified multiple times.</li> +specified multiple times. +Note that when $WorkDirectory is not set or set to a non-writable +location, the state file will not be generated.</li> <li><span style="font-weight: bold;">Facility facility</span><br> The syslog facility to be assigned to lines read. Can be specified in @@ -92,7 +94,7 @@ textual form (e.g. "info", "warning", ...) or as numbers (e.g. 4 for is "notice".</li> <li><b>PersistStateInterval</b> [lines]</b><br> Specifies how often the state file shall be written when processing the input -file. The default value is 0, which means a new state file is only written when +file. The <strong>default</strong> value is 0, which means a new state file is only written when the monitored files is being closed (end of rsyslogd execution). Any other value n means that the state file is written every time n file lines have been processed. This setting can be used to guard against message duplication due @@ -101,9 +103,9 @@ performance, especially when set to a low value. Frequently writing the state file is very time consuming. <li><b>ReadMode</b> [mode]</b><br> This mode should defined when having multiline messages. The value can range from 0-2 and determines the multiline detection method. -<br>0 (default) - line based (Each line is a new message) -<br>1 - indented (New log messages start at the beginning of a line. If a line starts with a space it is part of the log message before it) -<br>2 - paragraph (There is a blank line between log messages) +<br>0 (<strong>default</strong>) - line based (Each line is a new message) +<br>1 - paragraph (There is a blank line between log messages) +<br>2 - indented (New log messages start at the beginning of a line. If a line starts with a space it is part of the log message before it) <li><b>MaxLinesAtOnce</b> [number]</b> <br> This is useful if multiple files need to be monitored. If set to 0, each file @@ -112,11 +114,11 @@ will be fully processed and then processing switches to the next file [number] lines is processed in sequence for each file, and then the file is switched. This provides a kind of mutiplexing the load of multiple files and probably leads to a more natural distribution of events when multiple busy files -are monitored. The default is 1024. +are monitored. The <strong>default</strong> is 1024. <li><b>MaxSubmitAtOnce</b> [number]</b> <br> This is an expert option. It can be used to set the maximum input batch size that -imfile can generate. The default is 1024, which is suitable for a wide range of +imfile can generate. The <strong>default</strong> is 1024, which is suitable for a wide range of applications. Be sure to understand rsyslog message batch processing before you modify this option. If you do not know what this doc here talks about, this is a good indication that you should NOT modify the default. @@ -141,17 +143,17 @@ your distro puts rsyslog's config files). Note that only commands actually needed need to be specified. The second file uses less commands and uses defaults instead.<br> </p> -<textarea rows="15" cols="60">module(load="folder/to/rsyslog/plugins/imfile/.libs/imfile" PollingInterval="10") #needs to be done just once +<textarea rows="15" cols="60">module(load="imfile" PollingInterval="10") #needs to be done just once # File 1 input(type="imfile" File="/path/to/file1" -Tag="tag1" -StateFile="/var/spool/rsyslog/statefile1" -Severity="error" -Facility="local7") + Tag="tag1" + StateFile="statefile1" + Severity="error" + Facility="local7") # File 2 input(type="imfile" File="/path/to/file2" -Tag="tag2" -StateFile="/var/spool/rsyslog/statefile2") + Tag="tag2" + StateFile="statefile2") # ... and so on ... # </textarea> @@ -210,8 +212,7 @@ your distro puts rsyslog's config files). Note that only commands actually needed need to be specified. The second file uses less commands and uses defaults instead.<br> </p> -<textarea rows="15" cols="60">$ModLoad imfile # -needs to be done just once +<textarea rows="15" cols="60">$ModLoad imfile # needs to be done just once # File 1 $InputFileName /path/to/file1 $InputFileTag tag1: diff --git a/doc/imjournal.html b/doc/imjournal.html new file mode 100644 index 00000000..a4b232e8 --- /dev/null +++ b/doc/imjournal.html @@ -0,0 +1,111 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"><title>Systemd Journal Input Module</title></head> +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>Systemd Journal Input Module</h1> +<p><b>Module Name: imjournal</b></p> +<p><b>Author: </b>Milan Bartos +<mbartos@redhat.com></p> +<p><b>Description</b>:</p> +<p>Provides the ability to import structured log messages from systemd journal +to syslog.</p> +<p>Note that this module reads the journal database, what is considered a +relativly performance-intense operation. As such, the performance of a +configuration utilizing this +module may be notably slower then when using +<a href="imuxsock.html">imuxsock</a>. The journal provides imuxsock with a +copy of all "classical" syslog messages, however, it does not provide +structured data. If the latter is needed, imjournal must be used. Otherwise, +imjournal may be simply replaced by imuxsock. +<p>We suggest to check out our short presentation on +<a href="http://youtu.be/GTS7EuSdFKE">rsyslog journal integration</a> to +learn more details of anticipated use cases. + +<p><b>Warning:</b> Some versions of systemd journal have problems with database +corruption, which leads to the journal to return the same data endlessly +in a thight loop. This results in massive message duplication inside rsyslog +probably resulting in a denial-of-service when the system ressouces get +exhausted. This can be somewhat mitigated by using proper rate-limiters, but +even then there are spikes of old data which are endlessly repeated. By default, +ratelimiting is activated and permits to process 20,000 messages within 10 +minutes, what should be well enough for most use cases. If insufficient, use +the parameters described below to adjust the permitted volume. +<b>It is strongly recommended to use this plugin only if there +is hard need to do so.</b> + +<p><b>Configuration Directives</b>:</p> +<p><b>Module Directives</b></p> +<ul> +<li><b>PersistStateInterval</b> number-of-messages<br> +This is a global setting. It specifies how often should the journal state be persisted. +The persists happens after each <i>number-of-messages</i>. +This option is useful for rsyslog to start reding from the last journal message it read. + +<li><b>StateFile</b> /path/to/file<br> +This is a global setting. It specifies where the state file for persisting +journal state is located. + +<li><b>ratelimit.interval</b> seconds (default: 600)<br> +Specifies the interval in seconds onto which rate-limiting is to be applied. +If more than ratelimit.burst messages are read during that interval, further +messages up to the end of the interval are discarded. The number of messages +discarded is emitted at the end of the interval (if there were any discards). +<br>Setting this to value zero turns off ratelimiting. Note that it is +<b>not recommended to turn of ratelimiting</b>, except that you know for +sure journal database entries will never be corrupted. Without ratelimiting, +a corrupted systemd journal database may cause a kind of denial of service (we +are stressing this point as multiple users have reported us such problems +with the journal database - information current as of June 2013). + +<li><b>ratelimit.burst</b> messages (default: 20000)<br> +Specifies the maximum number of messages that can be emitted within the +ratelimit.interval interval. For futher information, see description there. + +<li><b>IgnorePreviousMessages</b> [<b>off</b>/on]<br> +This option specifies whether imjournal should ignore messages currently in +journal and read only new messages. This option is only used when there is +no StateFile to avoid message loss. +</ul> + +<b>Caveats/Known Bugs:</b> +<p> +<ul> +<li>As stated above, a corrupted systemd journal database can cause major +problems, depending on what the corruption results in. This is beyond the +control of the rsyslog team. +</ul> +</p> +<p><b>Sample:</b></p> +<p> +The following example shows pulling structured imjournal messages and saving them into /var/log/ceelog. +</p> +<textarea rows="11" cols="80"> +module(load="imjournal" PersistStateInterval="100" StateFile="/path/to/file") #load imjournal module +module(load="mmjsonparse") #load mmjsonparse module for structured logs + +template(name="CEETemplate" type="string" + string="%TIMESTAMP% %HOSTNAME% %syslogtag% @cee: %$!all-json%\n" + ) #template for messages + +action(type="mmjsonparse") +action(type="omfile" file="/var/log/ceelog" template="CEETemplate") +</textarea> + +<p><b>Legacy Configuration Directives</b>:</p> +<ul> +<li><b>$imjournalPersistStateInterval</b><br> +Equivalent to: PersistStateInterval</li> +<li><b>$imjournalStateFile</b><br> +Equivalent to: StateFile</li> +<li><b>$imjournalRatelimitInterval</b><br> +Equivalent to: ratelimit.interval</li> +<li><b>$imjournalRatelimitBurst</b><br> +Equivalent to: ratelimit.burst</li> +<li><strong>$ImjournalIgnorePreviousMessages</strong><br> +Equivalent to: ignorePreviousMessages</li> +</ul> + +</body> +</html> diff --git a/doc/imklog.html b/doc/imklog.html index 05292ddf..f2e36fd2 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -15,24 +15,68 @@ syslog engine.</p> <p><b>Configuration Directives</b>:</p> <ul> -<li><strong>$KLogInternalMsgFacility +<li><strong>LogPath</strong><br> +The path to the Kernel log. This value should only be changed if you really know what +you are doing.</li> +<li><strong>InternalMsgFacility <facility></strong><br> The facility which messages internally generated by imklog will have. imklog generates some messages of itself (e.g. on problems, startup and shutdown) and these do not stem from the kernel. Historically, under Linux, these too have "kern" facility. Thus, on Linux platforms the default is "kern" while on others it is "syslogd". You usually do not -need to specify this configuratin directive - it is included primarily +need to specify this configuration directive - it is included primarily for few limited cases where it is needed for good reason. Bottom line: if you don't have a good idea why you should use this setting, do not touch it.</li> -<li><span style="font-weight: bold;">$KLogPermitNonKernelFacility -[on/<span style="font-style: italic;">off</span>]<br> -</span>At least under BSD the kernel log may contain entries +<li><b>PermitNonKernelFacility [on/<i>off</i>]</b><br> +At least under BSD the kernel log may contain entries with non-kernel facilities. This setting controls how those are handled. The default is "off", in which case these messages are ignored. Switch it to on to submit non-kernel messages to rsyslog -processing.<span style="font-weight: bold;"></span></li> +processing.</li> +<li><b>ParseKernelTimeStamp</b> [on/<b>off</b>]<br> +If enabled and the kernel creates a timestamp for its log messages, this timestamp will be +parsed and converted into regular message time instead to use the receive time of the kernel +message (as in 5.8.x and before). Default is to not parse the kernel timestamp, because the +clock used by the kernel to create the timestamps is not supposed to be as accurate as the +monotonic clock required to convert it. Depending on the hardware and kernel, it can result +in message time differences between kernel and system messages which occurred at same time. +<li><b>KeepKernelTimeStamp</b> [on/<b>off</b>]<br> +If enabled, this option causes to keep the [timestamp] provided by the kernel at the begin +of in each message rather than to remove it, when it could be parsed and converted into +local time for use as regular message time. Only used when <b>ParseKernelTimestamp</b> is on. +<li><b>ConsoleLogLevel</b> [<i>number</i>] +(former klogd -c option) -- sets the console log level. If specified, only messages with +up to the specified level are printed to the console. The default is -1, which means that +the current settings are not modified. To get this behavior, do not specify +ConsoleLogLevel in the configuration file. Note that this is a global parameter. Each time +it is changed, the previous definition is re-set. The one activate will be that one that is +active when imklog actually starts processing. In short words: do not specify this +directive more than once! +</ul> +<b>Caveats/Known Bugs:</b> +<p>This is obviously platform specific and requires platform +drivers. +Currently, imklog functionality is available on Linux and BSD.</p> +<p>This module is <b>not supported on Solaris</b> and not needed there. +For Solaris kernel input, use <a href="imsolaris.html">imsolaris</a>.</p> +<p><b>Sample:</b></p> +<p>The following sample pulls messages from the kernel log. All +parameters are left by default, which is usually a good idea. Please +note that loading the plugin is sufficient to activate it. No directive +is needed to start pulling kernel messages.<br> +</p> +<textarea rows="4" cols="60">module(load="imklog") +</textarea> +<p><b>Legacy Configuration Directives</b>:</p> +<ul> +<li><strong>$KLogInternalMsgFacility +<facility></strong><br> +equivalent to: InternalMsgFacility</li> +<li><span style="font-weight: bold;">$KLogPermitNonKernelFacility +[on/<span style="font-style: italic;">off</span>]<br> +equivalent to: PermitNonKernelFacility</li> <li><span style="font-weight: bold;"></span>$DebugPrintKernelSymbols [on/<b>off</b>]<br> Linux only, ignored on other platforms (but may be specified)</li> @@ -50,14 +94,7 @@ it except if you have a very good reason. If you have one, let us know because otherwise new versions will no longer support it.<br> Linux only, ignored on other platforms (but may be specified)</li> <li><b>$klogConsoleLogLevel</b> [<i>number</i>] -(former klogd -c option) -- sets the console log level. If specified, only messages with -up to the specified level are printed to the console. The default is -1, which means that -the current settings are not modified. To get this behavior, do not specify -$klogConsoleLogLevel in the configuration file. Note that this is a global parameter. Each time -it is changed, the previous definition is re-set. The one activate will be that one that is -active when imklog actually starts processing. In short words: do not specify this -directive more than once! -<br><b>Linux only</b>, ignored on other platforms (but may be specified)</li> +<br>equivalent to: ConsoleLogLevel</li> <li><b>$klogUseSyscallInterface</b> [on/<b>off</b>] -- former klogd -s option<br> Linux only, ignored on other platforms (but may be specified)</li> @@ -65,40 +102,17 @@ Linux only, ignored on other platforms (but may be specified)</li> former klogd -2 option<br> Linux only, ignored on other platforms (but may be specified)<br style="font-weight: bold;"> </li> -<li><b>$klogParseKernelTimestamp</b> [on/<b>off</b>] -If enabled and the kernel creates a timestamp for its log messages, this timestamp will be -parsed and converted into regular message time instead to use the receive time of the kernel -message (as in 5.8.x and before). Default is to not parse the kernel timestamp, because the -clock used by the kernel to create the timestamps is not supposed to be as accurate as the -monotonic clock required to convert it. Depending on the hardware and kernel, it can result -in message time differences between kernel and system messages which occurred at same time. -</li> -<li><b>$klogKeepKernelTimestamp</b> [on/<b>off</b>] -If enabled, this option causes to keep the [timestamp] provided by the kernel at the begin -of in each message rather than to remove it, when it could be parsed and converted into -local time for use as regular message time. Only used, when $klogParseKernelTimestamp is on. -</li> +<li><b>$klogParseKernelTimeStamp</b> [on/<b>off</b>]<br> +equivalent to: ParseKernelTimeStamp</li> +<li><b>$klogKeepKernelTimeStamp</b> [on/<b>off</b>]<br> +equivalent to: KeepKernelTimeStamp</li> </ul> -<b>Caveats/Known Bugs:</b> -<p>This is obviously platform specific and requires platform -drivers. -Currently, imklog functionality is available on Linux and BSD.</p> -<p>This module is <b>not supported on Solaris</b> and not needed there. -For Solaris kernel input, use <a href="imsolaris.html">imsolaris</a>.</p> -<p><b>Sample:</b></p> -<p>The following sample pulls messages from the kernel log. All -parameters are left by default, which is usually a good idea. Please -note that loading the plugin is sufficient to activate it. No directive -is needed to start pulling kernel messages.<br> -</p> -<textarea rows="15" cols="60">$ModLoad imklog -</textarea> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2009 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2008-2012 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/imkmsg.html b/doc/imkmsg.html index 23b96147..ba73715d 100644 --- a/doc/imkmsg.html +++ b/doc/imkmsg.html @@ -16,7 +16,7 @@ Milan Bartos <p>Reads messages from the /dev/kmsg structured kernel log and submits them to the syslog engine.</p> <p> -The printk log buffer constains log records. These records are exported by /dev/kmsg +The printk log buffer contains log records. These records are exported by /dev/kmsg device as structured data in the following format:<br /> "level,sequnum,timestamp;<message text>\n"<br /> There could be continuation lines starting with space that contains key/value pairs.<br /> diff --git a/doc/impstats.html b/doc/impstats.html index 64b04a30..1da09ced 100644 --- a/doc/impstats.html +++ b/doc/impstats.html @@ -12,30 +12,70 @@ <p><b>Description</b>:</p> <p>This module provides periodic output of rsyslog internal counters. Note that the whole statistics system is currently under development. So -availabilty and format of counters may change and is not yet stable (so be +availability and format of counters may change and is not yet stable (so be prepared to change your trending scripts when you upgrade to a newer rsyslog version). <p>The set of available counters will be output as a set of syslog messages. This output is periodic, with the interval being configurable (default is 5 minutes). -Be sure that your configuration records the counter messages (default is syslog.info). +Be sure that your configuration records the counter messages (default is syslog.=info). +Besides logging to the regular syslog stream, the module can also be configured to +write statistics data into a (local) file. <p>Note that loading this module has impact on rsyslog performance. Depending on -settings, this impact may be noticable (for high-load environments). +settings, this impact may be noticeable (for high-load environments). <p>The rsyslog website has an updated overview of available <a href="http://rsyslog.com/rsyslog-statistic-counter/">rsyslog statistic counters</a>. </p> -<p><b>Configuration Directives</b>:</p> +<p><b>Module Confguration Parameters</b>:</p> +<p>This module supports module parameters, only. <ul> -<li>$PStatInterval <Seconds><br> -Sets the interval, in <b>seconds</b> at which messages are generated. Please note that the -actual interval may be a bit longer. We do not try to be precise and so the interval is -actually a sleep period which is entered after generating all messages. So the actual -interval is what is configured here plus the actual time required to generate messages. -In general, the difference should not really matter. -<li>$PStatFacility <numerical facility><br> -The numerical syslog facility code to be used for generated messages. Default -is 5 (syslog).This is useful for filtering messages.</li> -<li>$PStatSeverity <numerical severity><br> -The numerical syslog severity code to be used for generated messages. Default -is 6 (info).This is useful for filtering messages.</li> + <li><strong>interval </strong>[seconds] (default 300 [5minutes])<br> + Sets the interval, in <b>seconds</b> at which messages are generated. Please note that the + actual interval may be a bit longer. We do not try to be precise and so the interval is + actually a sleep period which is entered after generating all messages. So the actual + interval is what is configured here plus the actual time required to generate messages. + In general, the difference should not really matter. + <br></li> + <li><strong>facility </strong>[templateName]<br> + The numerical syslog facility code to be used for generated messages. Default + is 5 (syslog). This is useful for filtering messages. + <br></li> + <li><strong>severity </strong>[templateName]<br> + The numerical syslog severity code to be used for generated messages. Default + is 6 (info).This is useful for filtering messages. + <br></li> + <li><strong>format </strong>[json/cee/<b>legacy</b>](rsyslog v6.3.8+ only)<br> + Specifies the format of emitted stats messages. The default of "legacy" is + compatible with pre v6-rsyslog. The other options provide support for + structured formats (note the "cee" is actually "project lumberack" logging). + <br></li> + <li><strong>log.syslog </strong>[<b>on</b>/off] - available since 7.3.6<br> + This is a boolean setting specifying if data should be sent + to the usual syslog stream. This is useful if custom formatting + or more elaborate processing is desired. However, output is placed + under the same restrictions as regular syslog data, especially in + regard to the queue position (stats data may sit for an extended + period of time in queues if they are full).<br></li> + <li><strong>log.file </strong>[file name] - available since 7.3.6<br> + If specified, statistics data is written the specified file. For + robustness, this should be a local file. The file format cannot be + customized, it consists of a date header, followed by a colon, + followed by the actual statistics record, all on one line. Only + very limited error handling is done, so if things go wrong stats + records will probably be lost. Logging to file an be a useful + alternative if for some reasons (e.g. full queues) the regular + syslog stream method shall not be used solely. Note that turning + on file logging does NOT turn of syslog logging. If that is desired + log.syslog="off" must be explicitely set. + <br></li> + +</ul> +<p><b>Legacx Configuration Directives</b>:</p> +A limited set of parameters can also be set via the legacy configuration +syntax. Note that this is intended as an upward compatibilit layer, so +newer features are intentionally <b>not</b> available via legacy directives. +<ul> +<li>$PStatInterval <Seconds> - same as the "interval" parameter. +<li>$PStatFacility <numerical facility> - same as the "facility" parameter. +<li>$PStatSeverity <numerical severity> - same as the "severity" parameter. <li>$PStatJSON <on/<b>off</b>> (rsyslog v6.3.8+ only)<br> If set to on, stats messages are emitted as structured cee-enhanced syslog. If set to off, legacy format is used (which is compatible with pre v6-rsyslog). @@ -45,23 +85,45 @@ set to off, legacy format is used (which is compatible with pre v6-rsyslog). <ul> <li>This module MUST be loaded right at the top of rsyslog.conf, otherwise stats may not get turned on in all places.</li> -<li>experimental code</li> </ul> -<p><b>Sample:</b></p> +<p><b>Samples:</b></p> <p>This activates the module and records messages to /var/log/rsyslog-stats in 10 minute intervals:<br> </p> -<textarea rows="8" cols="60">$ModLoad impstats +<textarea rows="5" cols="60">module(load="impstats" interval="600" severity="7") + +# to actually gather the data: +syslog.=debug /var/log/rsyslog-stats +</textarea> +<p><b>Legacy Sample:</b></p> +<p>This activates the module and records messages to /var/log/rsyslog-stats in 10 minute intervals:</p> +<textarea rows="6" cols="60">$ModLoad impstats $PStatInterval 600 $PStatSeverity 7 -syslog.debug /var/log/rsyslog-stats +syslog.=debug /var/log/rsyslog-stats </textarea> +<p>In the next sample, the default interval of 5 minutes is used. However, this time +stats data is NOT emitted to the syslog stream but to a local file instead. +<p> +<textarea rows="3" cols="70">module(load="impstats" interval="600" severity="7" + log.syslog="off" /* need to turn log stream logging off! */ + log.file="/path/to/local/stats.log") +</textarea> +<p>And finally, we log to both the regular syslog log stream as well as a file. +Within the log stream, we forward the data records to another server: +<p> +<textarea rows="4" cols="70">module(load="impstats" interval="600" severity="7" + log.file="/path/to/local/stats.log") + +syslog.=debug @central.example.net +</textarea> + <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2010 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/imptcp.html b/doc/imptcp.html index 107dd306..41283e69 100644 --- a/doc/imptcp.html +++ b/doc/imptcp.html @@ -13,11 +13,9 @@ <p><b>Description</b>:</p> <p>Provides the ability to receive syslog messages via plain TCP syslog. This is a specialised input plugin tailored for high performance on Linux. It will -probably not run on any other platform. Also, it does no provide TLS services. +probably not run on any other platform. Also, it does not provide TLS services. Encryption can be provided by using <a href="rsyslog_stunnel.html">stunnel</a>. <p>This module has no limit on the number of listeners and sessions that can be used. -<p>Multiple receivers may be configured by -specifying $InputPTCPServerRun multiple times. </p> <p><b>Configuration Directives</b>:</p> @@ -33,7 +31,10 @@ globaly to all inputs defined by the module. Number of helper worker threads to process incoming messages. These threads are utilized to pull data off the network. On a busy system, additional helper threads (but not more than there are CPUs/Cores) can help improving -performance. The default value is two. +performance. The default value is two, which means there +is a default thread count of three (the main input thread plus two +helpers). +No more than 16 threads can be set (if tried to, rsyslog always resorts to 16). </ul> <p><b>Input Parameters</b>:</p> <p>These parameters can be used with the "input()" statement. They apply to the @@ -76,7 +77,7 @@ The default, 0, means that the operating system defaults are used. This has only effect if keep-alive is enabled. The functionality may not be available on all platforms. <li><b>KeepAlive.Interval</b> <number><br> -The interval between subsequential keepalive probes, regardless of what the connection has exchanged in the meantime. +The interval between subsequent keepalive probes, regardless of what the connection has exchanged in the meantime. The default, 0, means that the operating system defaults are used. This has only effect if keep-alive is enabled. The functionality may not be available on all platforms. @@ -95,6 +96,18 @@ the message was received from. Binds specified ruleset to next server defined. <li><b>Address</b> <name><br> On multi-homed machines, specifies to which local address the listerner should be bound. +<li><b>RateLimit.Interval</b> [number] - (available since 7.3.1) specifies the rate-limiting +interval in seconds. Default value is 0, which turns off rate limiting. Set it to a number +of seconds (5 recommended) to activate rate-limiting. +</li> +<li><b>RateLimit.Burst</b> [number] - (available since 7.3.1) specifies the rate-limiting +burst in number of messages. Default is 10,000. +<li><b>compression.mode</b><i>mode</i><br> +<i>mode</i> is one of "none" or "stream:always". +It is the counterpart to the compression modes set in +<a href="omfile.html">omfile</a>. +Please see it's documentation for details. +</li> </ul> <b>Caveats/Known Bugs:</b> <ul> @@ -130,11 +143,8 @@ Equivalent to: Port </li> Equivalent to: Name </li> <li>$InputPTCPServerBindRuleset <name><br> Equivalent to: Ruleset </li> -<li>$InputPTCPHelperThreads <number><br> -Number of helper worker threads to process incoming messages. These -threads are utilized to pull data off the network. On a busy system, additional -helper threads (but not more than there are CPUs/Cores) can help improving -performance. The default value is two. +<li>$InputPTCPServerHelperThreads <number><br> +Equivalent to: threads </li> <li>$InputPTCPServerListenIP <name><br> Equivalent to: Address </li> </ul> @@ -154,7 +164,7 @@ $InputPTCPServerRun 514 <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2010-2012 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2010-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/imrelp.html b/doc/imrelp.html index 856aff82..66e8b054 100644 --- a/doc/imrelp.html +++ b/doc/imrelp.html @@ -28,12 +28,80 @@ nits outlined above, is a much more reliable solution than plain tcp syslog and so it is highly suggested to use RELP instead of plain tcp. Clients send messages to the RELP server via omrelp.</p> -<p><b>Configuration Directives</b>:</p> +<p><b>Module Parameters</b>:</p> +<ul> + <li><b>Ruleset</b> <name></br> + Binds the specified ruleset to <b>all</b> RELP listeners. +</ul> +<p><b>Input Parameters</b>:</p> <ul> -<li><b>Ruleset</b> <name></br> -Binds the specified ruleset to all RELP listeners. <li><b>Port</b> <port><br> Starts a RELP server on selected port</li> +<li><b>tls</b> (not mandatory, values "on","off", default "off")<br> +If set to "on", the RELP connection will be encrypted by TLS, +so that the data is protected against observers. Please note +that both the client and the server must have set TLS to +either "on" or "off". Other combinations lead to unpredictable +results. +</li> +<li><b>tls.compression</b> (not mandatory, values "on","off", default "off")<br> +The controls if the TLS stream should be compressed (zipped). While this +increases CPU use, the network bandwidth should be reduced. Note that +typical text-based log records usually compress rather well. +</li> +<li><b>tls.dhbits</b> (not mandatory, integer)<br> +This setting controls how many bits are used for Diffie-Hellman key +generation. If not set, the librelp default is used. For secrity +reasons, at least 1024 bits should be used. Please note that the number +of bits must be supported by GnuTLS. If an invalid number is given, rsyslog +will report an error when the listener is started. We do this to be transparent +to changes/upgrades in GnuTLS (to check at config processing time, we would need +to hardcode the supported bits and keep them in sync with GnuTLS - this is +even impossible when custom GnuTLS changes are made...). +</li> +<li><b>tls.permittedPeer</b> peer</br> +Places access restrictions on this listener. Only peers which +have been listed in this parameter may connect. The validation +bases on the certificate the remote peer presents.<br> +The <i>peer</i> parameter lists permitted certificate +fingerprints. Note that it is an array parameter, so either +a single or multiple fingerprints can be listed. When a +non-permitted peer connects, the refusal is logged together +with it's fingerprint. So if the administrator knows this was +a valid request, he can simple add the fingerprint by copy and +paste from the logfile to rsyslog.conf. +<br>To specify multiple fingerprints, just enclose them +in braces like this: +<br>tls.permittedPeer=["SHA1:...1", "SHA1:....2"] +<br>To specify just a single peer, you can either +specify the string directly or enclose it in braces. +</li> +<li><b>tls.authMode</b> mode</br> +Sets the mode used for mutual authentication. Supported values are +either "<i>fingerprint</i>" or "<i>name"</i>. +<br>Fingerprint mode basically is what SSH +does. It does not require a full PKI to be present, instead self-signed +certs can be used on all peers. Even if a CA certificate is given, the +validity of the peer cert is NOT verified against it. Only the +certificate fingerprint counts. +<br>In "name" mode, certificate validation happens. Here, the matching +is done against the certificate's subjectAltName and, as a fallback, +the subject common name. If the certificate contains multiple names, +a match on any one of these names is considered good and permits the +peer to talk to rsyslog. +<li><b>tls.prioritystring</b> (not mandatory, string)<br> +This parameter permits to specify the so-called "priority string" to +GnuTLS. This string gives complete control over all crypto parameters, +including compression setting. For this reason, when the prioritystring +is specified, the "tls.compression" parameter has no effect and is +ignored. +<br>Full information about how to construct a priority string can be +found in the GnuTLS manual. At the time of this writing, this +information was contained in +<a href="http://gnutls.org/manual/html_node/Priority-Strings.html">section 6.10 of the GnuTLS manual</a>. +<br><b>Note: this is an expert parameter.</b> Do not use if you do +not exactly know what you are doing. +</li> </ul> <b>Caveats/Known Bugs:</b> <ul> @@ -47,7 +115,7 @@ not specific ones. This is due to a currently existing limitation in librelp. <p><b>Sample:</b></p> <p>This sets up a RELP server on port 20514.<br> </p> -<textarea rows="15" cols="60">module(load="/folder/to/rsyslog/plugins/imrelp/.libs/imrelp") # needs to be done just once +<textarea rows="5" cols="60">module(load="imrelp") # needs to be done just once input(type="imrelp" port="20514") </textarea> @@ -60,7 +128,6 @@ equivalent to: Port</li> </ul> <b>Caveats/Known Bugs:</b> <ul> -<li>see description</li> <li>To obtain the remote system's IP address, you need to have at least librelp 1.0.0 installed. Versions below it return the hostname instead of the IP address.</li> @@ -70,14 +137,14 @@ not specific ones. This is due to a currently existing limitation in librelp. <p><b>Sample:</b></p> <p>This sets up a RELP server on port 20514.<br> </p> -<textarea rows="15" cols="60">$ModLoad imrelp # needs to be done just once +<textarea rows="5" cols="60">$ModLoad imrelp # needs to be done just once $InputRELPServerRun 20514 </textarea> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/imsolaris.html b/doc/imsolaris.html index ce0e7e84..cc9a745f 100644 --- a/doc/imsolaris.html +++ b/doc/imsolaris.html @@ -18,7 +18,7 @@ is no special kernel input device. Instead, both kernel messages as well as messages emitted via syslog() are received from a single source. <p>This module obeys the Solaris door() mechanism to detect a running syslogd instance. As such, only one can be active at one time. If it detects another -active intance at startup, the module disables itself, but rsyslog will +active instance at startup, the module disables itself, but rsyslog will continue to run. <p><b>Configuration Directives</b>:</p> <ul> diff --git a/doc/imtcp.html b/doc/imtcp.html index feb0bdd7..5ac30d08 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -17,10 +17,6 @@ Encryption is natively provided by selecting the approprioate network stream driver and can also be provided by using <a href="rsyslog_stunnel.html">stunnel</a> (an alternative is the use the <a href="imgssapi.html">imgssapi</a> module).</p> -<p>Multiple receivers may be configured by specifying -$InputTCPServerRun multiple times. This is available since version 4.3.1, earlier -versions do NOT support it. -</p> <p><b>Configuration Directives</b>:</p> <p><b>Global Directives</b>:</p> @@ -67,13 +63,13 @@ the sender is throttled a bit when the queue becomes near-full. This is done in to preserve some queue space for inputs that can not throttle (like UDP), but it may have some undesired effect in some configurations. Still, we consider this as a useful setting and thus it is the default. To turn the handling off, simply -configure that explicitely. +configure that explicitly. </li> <li><b>MaxListeners</b> <number><br> Sets the maximum number of listeners (server ports) supported. Default is 20. This must be set before the first $InputTCPServerRun directive.</li> <li><b>MaxSessions</b> <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> <li><b>StreamDriver.Mode</b> <number><br> -Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specifc.</li> +Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specific.</li> <li><b>StreamDriver.AuthMode</b> <mode-string><br> Sets the authentication mode for the currently selected <a href="netstream.html">network stream driver</a>. <mode-string> is driver specifc.</li> <li><b>PermittedPeer</b> <id-string><br> @@ -100,6 +96,13 @@ activated. This is the default and should be left unchanged until you know very well what you do. It may be useful to turn it off, if you know this framing is not used and some senders emit multi-line messages into the message stream. </li> +<li><b>RateLimit.Interval</b> [number] - (available since 7.3.1) specifies the rate-limiting +interval in seconds. Default value is 0, which turns off rate limiting. Set it to a number +of seconds (5 recommended) to activate rate-limiting. +</li> +<li><b>RateLimit.Burst</b> [number] - (available since 7.3.1) specifies the rate-limiting +burst in number of messages. Default is 10,000. +</li> </ul> <b>Caveats/Known Bugs:</b> <ul> diff --git a/doc/imudp.html b/doc/imudp.html index 961bbeba..043d774c 100644 --- a/doc/imudp.html +++ b/doc/imudp.html @@ -37,7 +37,7 @@ processing under Linux (and thus reduce chance of packet loss). <li><b>SchedulingPriority</b> <number><br> Scheduling priority to use. </ul> -<p><b>Action Parameters</b>:</p> +<p><b>Input Parameters</b>:</p> <ul> <li><b>Address</b> <IP><br> local IP address (or name) the UDP listens should bind to</li> @@ -47,8 +47,35 @@ default 514, start UDP server on this port. Either a single port can be specifie <br>Array of ports: Port=["514","515","10514","..."]</li> <li><b>Ruleset</b> <ruleset><br> Binds the listener to a specific <a href="multi_ruleset.html">ruleset</a>.</li> -<li><b>rcvbufSize</b> <size><br> -Available since 7.5.3+<br> +<li><b>RateLimit.Interval</b> [number] - (available since 7.3.1) specifies the rate-limiting +interval in seconds. Default value is 0, which turns off rate limiting. Set it to a number +of seconds (5 recommended) to activate rate-limiting. +</li> +<li><b>RateLimit.Burst</b> [number] - (available since 7.3.1) specifies the rate-limiting +burst in number of messages. Default is 10,000. +</li> +<li><b>InputName</b> [name] - (available since 7.3.9) specifies the value of +the inputname. In older versions, this was always "imudp" for all listeners, +which still i the default. +Starting with 7.3.9 it can be set to different values for each listener. +Note that when a single input statement defines multipe listner ports, the +inputname will be the same for all of them. If you want to differentiate in that +case, use "InputName.AppendPort" to make them unique. +Note that the "InputName" parameter can be an empty string. In that case, the +corresponding inputname property will obviously also be the empty string. This +is primarily meant to be used togehter with "InputName.AppendPort" to set the +inputname equal to the port. +</li> +<li><b>InputName.AppendPort</b> [on/<b>off</b>] - (available since 7.3.9) +appends the port the the inputname. Note that when no inputname is specified, +the default of "imudp" is used and the port is appended to that default. So, +for example, a listner port of 514 in that case will lead to an inputname +of "imudp514". The ability to append a port is most useful when multiple ports +are defined for a single input and each of the inputnames shall be unique. +Note that there currently is no differentiation between IPv4/v6 listeners on +the same port. +</li> +<li><b>rcvbufSize</b> [size] - (available since 7.5.3) This request a socket receive buffer of specific size from the operating system. It is an expert parameter, which should only be changed for a good reason. Note that setting this parameter disables Linux auto-tuning, which usually works pretty well. @@ -68,20 +95,39 @@ privilege drop. This may be worked around by using a sufficiently-privileged user account. </li> </ul> -<p><b>Sample:</b></p> +<p><b>Samples:</b></p> <p>This sets up an UPD server on port 514:<br> </p> -<textarea rows="4" cols="60">module(load="imudp") # needs to be done just once +<textarea rows="3" cols="60">module(load="imudp") # needs to be done just once input(type="imudp" port="514") </textarea> <p>The following sample is mostly equivalent to the first one, but request a larger rcvuf size. Note that 1m most probably will not be honored by the OS until the user is sufficiently privileged.</p> -<textarea rows="4" cols="60">module(load="imudp") # needs to be done just once +<textarea rows="3" cols="60">module(load="imudp") # needs to be done just once input(type="imudp" port="514" rcvbufSize="1m") </textarea> +<p>In the next example, we set up three listeners at ports 10514, 10515 and 10516 +and assign a listner name of "udp" to it, followed by the port number: +</p> +<textarea rows="4" cols="60">module(load="imudp") +input(type="imudp" port=["10514","10515","10516"] + inputname="udp" inputname.appendPort="on") +</textarea> + +<p>The next example is almost equal to the previous one, but +now the inputname property will just be set to the port number. +So if a message was received on port 10515, the input name will be +"10515" in this example whereas it was "udp10515" in the previous one. +Note that to do that we set the inputname to the empty string. +</p> +<textarea rows="4" cols="60">module(load="imudp") +input(type="imudp" port=["10514","10515","10516"] + inputname="" inputname.appendPort="on") +</textarea> + <p><b>Legacy Configuration Directives</b>:</p> <p>Multiple receivers may be configured by specifying $UDPServerRun multiple times. @@ -103,16 +149,17 @@ equivalent to: SchedulingPriority <p><b>Legacy Sample:</b></p> <p>This sets up an UPD server on port 514:<br> </p> -<textarea rows="4" cols="60">$ModLoad imudp # needs to be done just once +<textarea rows="3" cols="60">$ModLoad imudp # needs to be done just once $UDPServerRun 514 </textarea> + <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2009-2013 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +Copyright © 2009-2013 by +<a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body></html> diff --git a/doc/imuxsock.html b/doc/imuxsock.html index d505604a..80c3bc5a 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -42,7 +42,7 @@ severity and configure things accordingly. To turn off rate limiting, set the interval to zero. <p><b>Unix log sockets can be flow-controlled.</b> That is, if processing queues fill up, the unix socket reader is blocked for a short while. This may be useful to prevent overruning -the queues (which may cause exessive disk-io where it actually would not be needed). However, +the queues (which may cause excessive disk-io where it actually would not be needed). However, flow-controlling a log socket (and especially the system log socket) can lead to a very unresponsive system. As such, flow control is disabled by default. That means any log records are places as quickly as possible into the processing queues. If you would like to have @@ -65,6 +65,12 @@ you must turn it on (via SysSock.Annotate and Annotate). <li><b>SysSock.IgnoreTimestamp</b> [<b>on</b>/off]<br> Ignore timestamps included in the messages, applies to messages received via the system log socket. </li> +<li><b>SysSock.IgnoreOwnMessages</b> [<b>on</b>/off] (available since 7.3.7)<br> +Ignores messages that originated from the same instance of rsyslogd. There usually +is no reason to receive messages from ourselfs. This setting is vital +when writing messages to the Linux journal. See <a href="omjournal.html">omjournal</a> +module documentation for a more in-depth description. +</li> <li><b>SysSock.Use</b> (imuxsock) [on/<b>off</b>] do NOT listen for the local log socket. This is most useful if you run multiple instances of rsyslogd where only one shall handle the system log socket. @@ -95,18 +101,30 @@ properties for the system log socket.</li> <li><b>SysSock.ParseTrusted</b> <on/<b>off</b>> if Annotation is turned on, create JSON/lumberjack properties out of the trusted properties (which can be accessed via RainerScript JSON Variables, e.g. "$!pid") instead of adding them to the message. -properties for the system log socket.</li> +</li> +<li><b>SysSock.Unlink</b> <<b>on</b>/off> (available since 7.3.9)<br> +if turned on (default), the system socket is unlinked and re-created when +opened and also unlinked when finally closed. Note that this setting has +no effect when running under systemd control (because systemd handles +the socket). +</li> </ul> <p><b>Input Instance Parameters</b></p> <ul> <li><b>IgnoreTimestamp</b> [<b>on</b>/off] <br>Ignore timestamps included in the message. Applies to the next socket being added.</li> +<li><b>IgnoreOwnMessages</b> [<b>on</b>/off] (available since 7.3.7)<br> +Ignore messages that originated from the same instance of rsyslogd. There usually +is no reason to receive messages from ourselfs. This setting is vital +when writing messages to the Linux journal. See <a href="omjournal.html">omjournal</a> +module documentation for a more in-depth description. +</li> <li><b>FlowControl</b> [on/<b>off</b>] - specifies if flow control should be applied to the next socket.</li> <li><b>RateLimit.Interval</b> [number] - specifies the rate-limiting interval in seconds. Default value is 0, which turns off rate limiting. Set it to a number -of seconds (5 recommended) to activate rate-limiting. The default of 0 has been choosen +of seconds (5 recommended) to activate rate-limiting. The default of 0 has been chosen as people experienced problems with this feature activated by default. Now it needs an explicit opt-in by setting this parameter. </li> @@ -126,7 +144,7 @@ be obtained from the log socket itself. If so, the TAG part of the message is re It is recommended to turn this option on, but the default is "off" to keep compatible with earlier versions of rsyslog. </li> <li><b>UseSysTimeStamp</b> [<b>on</b>/off] instructs imuxsock -to obtain message time from the system (via control messages) insted of using time +to obtain message time from the system (via control messages) instead of using time recorded inside the message. This may be most useful in combination with systemd. Note: this option was introduced with version 5.9.1. Due to the usefulness of it, we decided to enable it by default. As such, 5.9.1 and above behave slightly different @@ -154,6 +172,13 @@ that the local hostname can be overridden in cases where that is desired.</li> properties for the non-system log socket in question.</li> <li><b>ParseTrusted</b> <on/<b>off</b>> equivalent to the SysSock.ParseTrusted module parameter, but applies to the input that is being defined. +<li><b>Unlink</b> <<b>on</b>/off> (available since 7.3.9)<br> +if turned on (default), the socket is unlinked and re-created when +opened and also unlinked when finally closed. Set it to off if you +handle socket creation yourself. Note that handling socket creation +oneself has the advantage that a limited amount of messages may be +queued by the OS if rsyslog is not running. +</li> </ul> <b>Caveats/Known Bugs:</b><br> diff --git a/doc/licensing.html b/doc/licensing.html index 93a50930..0a4ab1f4 100644 --- a/doc/licensing.html +++ b/doc/licensing.html @@ -22,7 +22,7 @@ real details, check source files and the files COPYING and COPYING.LESSER inside <p>Each of these components can be thought of as individual projects. In fact, some of the plugins have different main authors than the rest of the rsyslog package. All of these components are currently put together into a single "rsyslog" package (tarball) for -convinience: this makes it easier to distribute a consistent version where everything +convenience: this makes it easier to distribute a consistent version where everything is included (and in the right versions) to build a full system. Platform package maintainers in general take the overall package and split off the individual components, so that users can install only what they need. In source installations, this can be done via the diff --git a/doc/log_rotation_fix_size.html b/doc/log_rotation_fix_size.html index 51edf033..7cdecc6c 100644 --- a/doc/log_rotation_fix_size.html +++ b/doc/log_rotation_fix_size.html @@ -31,7 +31,7 @@ Channels to achieve this. Putting the following directive</p> <p><pre> # start log rotation via outchannel -# outchannel definiation +# outchannel definition $outchannel log_rotation,/var/log/log_rotation.log, 52428800,/home/me/./log_rotation_script # activate the channel and log everything to it *.* :omfile:$log_rotation @@ -49,13 +49,13 @@ mv -f /var/log/log_rotation.log /var/log/log_rotation.log.1 <p>This moves the original log to a kind of backup log file. After the action was successfully performed rsyslog creates a new /var/log/log_rotation.log -file and fill it up with new logs. So the latest logs are always in log_roatation.log.</p> +file and fill it up with new logs. So the latest logs are always in log_rotation.log.</p> <h2>Conclusion</h2> <p>With this approach two files for logging are used, each with a maximum size of 50 MB. So we can say we have successfully configured a log rotation which satisfies our requirement. -We keep the logs at a fixed-size level of100 MB.</p> +We keep the logs at a fixed-size level of 100 MB.</p> <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> diff --git a/doc/lookup_tables.html b/doc/lookup_tables.html new file mode 100644 index 00000000..d72810f1 --- /dev/null +++ b/doc/lookup_tables.html @@ -0,0 +1,205 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<title>Lookup Tables</title> +</head> + +<body> +<h1>Lookup Tables</h1> + +<p><b><font color="red">NOTE: this is</font> proposed functionality, which is +<font color="red">NOT YET IMPLEMENTED</font>!</b> + +<p><b>Lookup tables</a> are a powerful construct +to obtain "class" information based on message content (e.g. to build +log file names for different server types, departments or remote +offices).</b> +<p>The base idea is to use a message variable as an index into a table which then +returns another value. For example, $fromhost-ip could be used as an index, with +the table value representing the type of server or the department or remote office +it is located in. A main point with lookup tables is that the lookup is very fast. +So while lookup tables can be emulated with if-elseif constructs, they are generally +much faster. Also, it is possible to reload lookup tables during rsyslog runtime without +the need for a full restart. +<p>The lookup tables itself exists in a separate configuration file (one per table). This +file is loaded on rsyslog startup and when a reload is requested. +<p>There are different types of lookup tables: +<ul> +<li><b>string</b> - the value to be looked up is an arbitrary string. Only exact +some strings match. +<li><b>array</b> - the value to be looked up is an integer number from a consequtive set. +The set does not need to start at zero or one, but there must be no number missing. So, for example +5,6,7,8,9 would be a valid set of index values, while 1,2,4,5 would not be (due to missing +2). +A match happens if the requested number is present. +<li><b>sparseArray</b> - the value to be looked up is an integer value, but there may +be gaps inside the set of values (usually there are large gaps). A typical use case would +be the matching of IPv4 address information. A match happens on the first value that is +less than or equal to the requested value. +</ul> +<p>Note that index integer numbers are represented by unsigned 32 bits. +<p>Lookup tables can be access via the lookup() built-in function. The core idea is to +set a local variable to the lookup result and later on use that local variable in templates. +<p>More details on usage now follow. +<h2>Lookup Table File Format</h2> +<p>Lookup table files contain a single JSON object. This object contains of a header and a +table part. +<h3>Header</h3> +<p>The header is the top-level json. It has paramters "version", "nomatch", and "type". +The version parameter +must be given and must always be one for this version of rsyslog. The nomatch +parameter is optional. If specified, it contains the value to be used if lookup() +is provided an index value for which no entry exists. The default for +"nomatch" is the empty string. Type specifies the type of lookup to be done. +<h3>Table</h3> +This must be an array of elements, even if only a single value exists (for obvious +reasons, we do not expect this to occur often). Each array element must contain two +fields "index" and "value". +<h3>Example</h3> +<p>This is a sample of how an ip-to-office mapping may look like: +<pre> +{ "version":1, "nomatch":"unk", "type":"string", + "table":[ {"index":"10.0.1.1", "value":"A" }, + {"index":"10.0.1.2", "value":"A" }, + {"index":"10.0.1.3", "value":"A" }, + {"index":"10.0.2.1", "value":"B" }, + {"index":"10.0.2.2", "value":"B" }, + {"index":"10.0.2.3", "value":"B" } + ] +} +</pre> +Note: if a different IP comes in, the value "unk" +is returend thanks to the nomatch parameter in +the first line. +<p> +<h2>RainerScript Statements</h2> +<h3>lookup_table() Object</h3> +<p>This statement defines and intially loads a lookup table. Its format is +as follows: +<pre> +lookup_table(name="name" file="/path/to/file" reloadOnHUP="on|off") +</pre> +<h4>Parameters</h4> +<ul> + <li><b>name</b> (mandatory)<br> + Defines the name of lookup table for further reference + inside the configuration. Names must be unique. Note that + it is possible, though not advisible, to have different + names for the same file. + <li><b>file</b> (mandatory)<br> + Specifies the full path for the lookup table file. This file + must be readable for the user rsyslog is run under (important + when dropping privileges). It must point to a valid lookup + table file as described above. + <li><b>reloadOnHUP</b> (optional, default "on")<br> + Specifies if the table shall automatically be reloaded + as part of HUP processing. For static tables, the + default is "off" and specifying "on" triggers an + error message. Note that the default of "on" may be + somewhat suboptimal performance-wise, but probably + is what the user intuitively expects. Turn it off + if you know that you do not need the automatic + reload capability. +</ul> + +<h3>lookup() Function</h3> +<p>This function is used to actually do the table lookup. Format: +<pre> +lookup_table("name", indexvalue) +</pre> +<h4>Parameters</h4> +<ul> + <li><b>return value</b><br> + The function returns the string that is associated with the + given indexvalue. If the indexvalue is not present inside the + lookup table, the "nomatch" string is returned (or an empty string + if it is not defined). + <li><b>name</b> (constant string)<br> + The lookup table to be used. Note that this must be specificed as a + constant. In theory, variable table names could be made possible, but + their runtime behaviour is not as good as for static names, and we do + not (yet) see good use cases where dynamic table names could be useful. + <li><b>indexvalue</b> (expression)<br> + The value to be looked up. While this is an arbitrary RainerScript expression, + it's final value is always converted to a string in order to conduct + the lookup. For example, "lookup(table, 3+4)" would be exactly the same + as "lookup(table, "7")". In most cases, indexvalue will probably be + a single variable, but it could also be the result of all RainerScript-supported + expression types (like string concatenation or substring extraction). + Valid samples are "lookup(name, $fromhost-ip & $hostname)" or + "lookup(name, substr($fromhost-ip, 0, 5))" as well as of course the + usual "lookup(table, $fromhost-ip)". +</ul> + + +<h3>load_lookup_table Statement</h3> + +<p><b>Note: in the final implementation, this MAY be implemented as an action. +This is a low-level decesion that must be made during the detail development +process. Parameters and semantics will remain the same of this happens.</b> + +<p>This statement is used to reload a lookup table. It will fail if +the table is static. While this statement is executed, lookups to this table +are temporarily blocked. So for large tables, there may be a slight performance +hit during the load phase. It is assume that always a triggering condition +is used to load the table. +<pre> +load_lookup_table(name="name" errOnFail="on|off" valueOnFail="value") +</pre> +<h4>Parameters</h4> +<ul> + <li><b>name</b> (string)<br> + The lookup table to be used. + <li><b>errOnFail</b> (boolean, default "on")<br> + Specifies whether or not an error message is to be emitted if + there are any problems reloading the lookup table. + <li><b>valueOnFail</b> (optional, string)<br> + This parameter affects processing if the lookup table cannot + be loaded for some reason: If the parameter is not present, + the previous table will be kept in use. If the parameter is + given, the previous table will no longer be used, and instead + an empty table be with nomath=valueOnFail be generated. In short, + that means when the parameter is set and the reload fails, + all matches will always return what is specified in valueOnFail. +</ul> + +<h3>Usage example</h3> +<p>For clarity, we show only those parts of rsyslog.conf that affect +lookup tables. We use the remote office example that an example lookup +table file is given above for. +<pre> +lookup_table(name="ip2office" file="/path/to/ipoffice.lu" + reloadOnHUP="off") + + +template(name="depfile" type="string" + string="/var/log/%$usr.dep%/messages") + +set $usr.dep = lookup("ip2office", $fromhost-ip); +action(type="omfile" dynfile="depfile") + +# support for reload "commands" +if $fromhost-ip == "10.0.1.123" + and $msg contains "reload office lookup table" + then + load_lookup_table(name="ip2office" errOnFail="on") +</pre> + +<p>Note: for performance reasons, it makes sense to put the reload command into +a dedicated ruleset, bound to a specific listener - which than should also +be sufficiently secured, e.g. via TLS mutual auth. + +<h2>Implementation Details</h2> +<p>The lookup table functionality is implemented via highly efficient algorithms. +The string lookup is based on a parse tree and has O(1) time complexity. The array +lookup is also O(1). In case of sparseArray, we have O(log n). +<p>To preserve space and, more important, increase cache hit performance, equal +data values are only stored once, no matter how often a lookup index points to them. +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/manual.html b/doc/manual.html index fc14031f..8d5eb733 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -13,13 +13,13 @@ It is quite compatible to stock sysklogd and can be used as a drop-in replacement. Its <a href="features.html"> advanced features</a> make it suitable for enterprise-class, <a href="rsyslog_tls.html">encryption protected syslog</a> relay chains while at the same time being very easy to setup for the -novice user. And as we know what enterprise users really need, there is -also <a href="http://www.rsyslog.com/professional-services">professional -rsyslog support</a> available directly from the source!</p> +novice user. And as we know what enterprise users really need, there are +also <a href="http://www.rsyslog.com/professional-services"> rsyslog +professional services</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 7.2.7 (v7.2-stable branch) of rsyslog.</b> +<p><b>This documentation is for version 7.5.2 (devel branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might diff --git a/doc/messageparser.html b/doc/messageparser.html index 370db59f..d22021dd 100644 --- a/doc/messageparser.html +++ b/doc/messageparser.html @@ -23,7 +23,7 @@ the rsyslog code). like input and output modules). That means that new message parsers can be added without modifying the rsyslog core, even without contributing something back to the project. -<p>But that doesn't answer what a message parser really is. What does ist mean to "parse a +<p>But that doesn't answer what a message parser really is. What does it mean to "parse a message" and, maybe more importantly, what is a message? To answer these questions correctly, we need to dig down into the relevant standards. <a href="http://tools.ietf.org/html/rfc5424">RFC5424</a> specifies a layered architecture @@ -49,7 +49,7 @@ reason) a single message into two and encapsulates these into two frames, there a message parser could undo that. <p>A typical example may be a multi-line message: let's assume some originator has generated a message for the format "A\nB" (where \n means LF). If that message is being transmitted -via plain tcp syslog, the frame delimiter is LF. So the sender will delimite the frame with +via plain tcp syslog, the frame delimiter is LF. So the sender will delimit the frame with LF, but otherwise send the message unmodified onto the wire (because that is how things are -unfortunately- done in plain tcp syslog...). So wire will see "A\nB\n". When this arrives at the receiver, the transport layer will undo the framing. When it sees the LF @@ -58,7 +58,7 @@ the receive will extract one complete message A and one complete message B, not that they once were both part of a large multi-line message. These two messages are then passed to the upper layers, where the message parsers receive them and extract information. However, the message parsers never know (or even have a chance to see) that A and B -belonged together. Even further, in rsyslog there is no guarnatee that A will be parsed +belonged together. Even further, in rsyslog there is no guarantee that A will be parsed before B - concurrent operations may cause the reverse order (and do so very validly). <p>The important lesson is: <b>message parsers can not be used to fix a broken framing</b>. You need a full protocol implementation to do that, what is the domain of input and @@ -73,10 +73,10 @@ the real-world evil that you can usually see. So I won't repeat that here. But i real problem is not the framing, but how to make malformed messages well-looking. <p><b>This is what message parsers permit you to do: take a (well-known) malformed message, parse it according to its semantics and generate perfectly valid internal message representations -from it.</b> So as long as messages are consistenly in the same wrong format (and they usually +from it.</b> So as long as messages are consistently in the same wrong format (and they usually are!), a message parser can look at that format, parse it, and make the message processable just -like it were wellformed in the first place. Plus, one can abuse the interface to do some other -"intersting" tricks, but that would take us to far. +like it were well formed in the first place. Plus, one can abuse the interface to do some other +"interesting" tricks, but that would take us to far. <p>While this functionality may not sound exciting, it actually solves a very big issue (that you only really understand if you have managed a system with various different syslog sources). Note that we were often able to process malformed messages in the past with the help of the @@ -113,15 +113,15 @@ interface probably extended, to support generic filter modules. These would need to the root of the parser chain. As mentioned, the current system already supports this. <p>The position inside the parser chain can be thought of as a priority: parser sitting earlier in the chain take precedence over those sitting later in it. So more specific -parser should go ealier in the chain. A good example of how this works is the default parser +parser should go earlier in the chain. A good example of how this works is the default parser set provided by rsyslog: rsyslog.rfc5424 and rsyslog.rfc3164, each one parses according to the rfc that has named it. RFC5424 was designed to be distinguishable from RFC3164 message by the sequence "1 " immediately after the so-called PRI-part (don't worry about these words, it is -sufficient if you understand there is a well-defined sequence used to indentify RFC5424 +sufficient if you understand there is a well-defined sequence used to identify RFC5424 messages). In contrary, RFC3164 actually permits everything as a valid message. Thus the RFC3164 parser will always parse a message, sometimes with quite unexpected outcome (there is a lot of guesswork involved in that parser, which unfortunately is unavoidable due to -existing techology limits). So the default parser chain is to try the RFC5424 parser first +existing technology limits). So the default parser chain is to try the RFC5424 parser first and after it the RFC3164 parser. If we have a 5424-formatted message, that parser will identify and parse it and the rsyslog engine will stop processing. But if we receive a legacy syslog message, the RFC5424 will detect that it can not parse it, return this status @@ -139,16 +139,16 @@ case, rsyslog has no other choice than to discard the message. If it does so, it a warning message, but only in the first 1,000 incidents. This limit is a safety measure against message-loops, which otherwise could quickly result from a parser chain misconfiguration. <b>If you do not tolerate loss of unparsable messages, you must ensure -that each message can be parsed.</b> You can easily achive this by always using the +that each message can be parsed.</b> You can easily achieve this by always using the "rsyslog-rfc3164" parser as the <i>last</i> parser inside parser chains. That may result in invalid parsing, but you will have a chance to see the invalid message (in debug mode, a warning message will be written to the debug log each time a message is dropped due to inability to parse it). <h3>Where are parser chains used?</h3> <p>We now know what parser chains are and how they operate. The question is now how many -parser chains can be active and how it is decicded which parser chain is used on which message. +parser chains can be active and how it is decided which parser chain is used on which message. This is controlled via <a href="multi_ruleset.html">rsyslog's rulesets</a>. In short, multiple -rulesets can be defined and there always exist at least one ruleset (for specifcs, follow +rulesets can be defined and there always exist at least one ruleset (for specifics, follow the <a href="multi_ruleset.html">link</a>). A parser chain is bound to a specific ruleset. This is done by virtue of defining parsers via the <a href="rsconf1_rulesetparser.html">$RulesetParser</a> configuration directive (for specifics, @@ -161,22 +161,22 @@ is added to the end of the (initially empty) ruleset's parser chain. <p>The correct answer is: generally yes, but it depends. First of all, remember that input modules (and specific listeners) may be bound to specific rulesets. As parser chains "reside" in rulesets, binding to a ruleset also binds to the parser chain that is bound to that ruleset. -As a number one prequisite, the input module must support binding to different rulesets. Not +As a number one prerequisite, the input module must support binding to different rulesets. Not all do, but their number is growing. For example, the important <a href="imudp.html">imudp</a> and <a href="imtcp.html">imtcp</a> input modules support that functionality. Those that do not (for example <a href="im3195">im3195</a>) can only utilize the default ruleset and thus the parser chain defined in that ruleset. <p>If you do not know if the input module in question supports ruleset binding, check -its documentation page. Those that support it have the requiered directives. +its documentation page. Those that support it have the required directives. <p>Note that it is currently under evaluation if rsyslog will support binding parser chains to specific inputs directly, without depending on the ruleset. There are some concerns that this may not be necessary but adds considerable complexity to the configuration. So this may or may not be possible in the future. In any case, if we decide to add it, input modules need to support it, so this functionality would require some time to implement. -<p>The coockbook recipe for using different parsers for different devices is given +<p>The cookbook recipe for using different parsers for different devices is given as an actual in-depth example in the <a href="rscon1_rulesetsparser.html">$RulesetParser</a> -configuration directive doc page. In short, it is acomplished by defining specific rulesets -for the required parser chains, definining different listener ports for each of the devices +configuration directive doc page. In short, it is accomplished by defining specific rulesets +for the required parser chains, defining different listener ports for each of the devices with different format and binding these listeners to the correct ruleset (and thus parser chains). Using that approach, a variety of different message formats can be supported via a single rsyslog instance. @@ -185,19 +185,19 @@ via a single rsyslog instance. <p>As of this writing, there exist only two message parsers, one for RFC5424 format and one for legacy syslog (loosely described in <a href="http://tools.ietf.org/html/rfc3164">RFC3164</a>). These parsers are built-in and -must not be explicitely loaded. However, message parsers can be added with relative ease +must not be explicitly loaded. However, message parsers can be added with relative ease by anyone knowing to code in C. Then, they can be loaded via $ModLoad just like any other loadable module. It is expected that the rsyslog project will be contributed additional message parsers over time, so that at some point there hopefully is a rich choice of them (I intend to add a browsable repository as soon as new parsers pop up). <h3>How to write a message parser?</h3> -<p>As a prequisite, you need to know the exact format that the device is sending. Then, you need +<p>As a prerequisite, you need to know the exact format that the device is sending. Then, you need moderate C coding skills, and a little bit of rsyslog internals. I guess the rsyslog specific part should not be that hard, as almost all information can be gained from the existing parsers. They are rather simple in structure and can be found under the "./tools" directory. They are named pmrfc3164.c and pmrfc5424.c. You need to follow the usual loadable module guidelines. It is my expectation that writing a parser should typically not take longer than a single -day, with maybe a day more to get aquainted with rsyslog. Of course, I am not sure if the number +day, with maybe a day more to get acquainted with rsyslog. Of course, I am not sure if the number is actually right. <p>If you can not program or have no time to do it, Adiscon can also write a message parser for you as @@ -209,7 +209,7 @@ provide a fast and efficient solution for this problem. Different parsers can be different devices, and they all convert message information into rsyslog's well-defined internal format. Message parsers were first introduced in rsyslog 5.3.4 and also offer some interesting ideas that may be explored in the future - up to full message normalization -capabilities. It is strongly recommended that anyone with a heterogenous environment take +capabilities. It is strongly recommended that anyone with a heterogeneous environment take a look at message parser capabilities. <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual diff --git a/doc/mmanon.html b/doc/mmanon.html new file mode 100644 index 00000000..16065a1f --- /dev/null +++ b/doc/mmanon.html @@ -0,0 +1,119 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>IP Address Anonimization Module (mmanon)</title></head> + +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>IP Address Anonimization Module (mmanon)</h1> +<p><b>Module Name: mmanon</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available since</b>: 7.3.7</p> +<p><b>Description</b>:</p> +<p>The mmanon module permits to anonymize IP addresses. It is a message +modification module that actually changes the IP address inside the message, +so after calling mmanon, the original message can no longer be obtained. +Note that anonymization will break digital signatures on the message, if +they exist. +<p><i>How are IP-Addresses defined?</i> +<p>We assume that an IP address consists of four octets in dotted notation, +where each of the octets has a value between 0 and 255, inclusively. After +the last octet, there must be either a space or a colon. So, for example, +"1.2.3.4 Test" and "1.2.3.4:514 Test" are detected as containing valid IP +addresses, whereas this is not the case for "1.2.300.4 Test" or +"1.2.3.4-Test". The message text may contain multiple addresses. If so, +each of them is anonimized (according to the same rules). +<b>Important:</b> We may change the set of acceptable characters after +the last octet in the future, if there are good reasons to do so. +<p> </p> + +<p><b>Module Configuration Parameters</b>:</p> +<p>Currently none. +<p> </p> +<p><b>Action Confguration Parameters</b>:</p> +<ul> +<li><b>mode</b> - default "rewrite"<br> +There exists the "simple" and "rewrite" mode. In simple mode, only octets +as whole can be anonymized and the length of the message is never changed. +This means that when the last three octets of the address 10.1.12.123 are +anonymized, the result will be 10.0.00.000. This means that the length of the +original octets is still visible and may be used to draw some privacy-evasive +conclusions. This mode is slightly faster than "overwrite" mode, and this +may matter in high throughput environments.<br> +The default "rewrite" mode will do full anonymization of any number of bits +and it will also normlize the address, so that no information about the +original IP address is available. So in the above example, 10.1.12.123 would +be anonymized to 10.0.0.0. +<li><b>ipv4.bits</b> - default 16<br> +This set the number of bits that should be anonymized (bits are from the +right, so lower bits are anonymized first). This setting permits to save +network information while still anonymizing user-specific data. The more +bits you discard, the better the anonymization obviously is. The default +of 16 bits reflects what German data privacy rules consider as being +sufficinetly anonymized. We assume, this can also be used as a rough +but conservative guideline for other countries.<br> +Note: when in simple mode, only bits on a byte boundary can be specified. +As such, any value other than 8, 16, 24 or 32 is invalid. If an invalid +value is given, it is rounded to the next byte boundary (so we favor stronger +anonymization in that case). For example, a bit value of 12 will become 16 in +simple mode (an error message is also emitted). +<li><b>replacementChar</b> - default "x"<br> +In simple mode, this sets the character +that the to-be-anonymized part of the IP address is to be overwritten +with. In rewrite mode, this parameter is <b>not permitted</b>, as in +this case we need not necessarily rewrite full octets. As such, the anonymized +part is always zero-filled and replacementChar is of no use. If it is +specified, an error message is emitted and the parameter ignored. +</ul> + +<p><b>Caveats/Known Bugs:</b> +<ul> +<li><b>only IPv4</b> is supported +</ul> + +<p><b>Samples:</b></p> +<p>In this snippet, we write one file without anonymization and another one +with the message anonymized. Note that once mmanon has run, access to the +original message is no longer possible (execept if stored in user +variables before anonymization). +<p><textarea rows="5" cols="60">module(load="mmanon") +action(type="omfile" file="/path/to/non-anon.log") +action(type="mmanon") +action(type="omfile" file="/path/to/anon.log") +</textarea> + +<p>This next snippet is almost identical to the first one, but +here we anonymize the full IPv4 address. Note that by +modifying the number of bits, you can anonymize different parts +of the address. Keep in mind that in simple mode (used here), the bit values +must match IP address bytes, so for IPv4 only the values 8, 16, 24 and +32 are valid. Also, in this example the replacement is done +via asterisks instead of lower-case "x"-letters. Also keep in mind that +"replacementChar" can only be set in simple mode. +<p><textarea rows="5" cols="60">module(load="mmanon") +action(type="omfile" file="/path/to/non-anon.log") +action(type="mmanon" ipv4.bits="32" mode="simple" replacementChar="*") +action(type="omfile" file="/path/to/anon.log") +</textarea> + +<p>The next snippet is also based on the first one, but anonimzes an +"odd" number of bits, 12. The value of 12 is used by some folks as a +compromise between keeping privacy and still permiting to gain some +more in-depth insight from log files. Note that anonymizing 12 bits +may be insufficient to fulfill legal requirements (if such exist). +<p><textarea rows="5" cols="60">module(load="mmanon") +action(type="omfile" file="/path/to/non-anon.log") +action(type="mmanon" ipv4.bits="12") +action(type="omfile" file="/path/to/anon.log") +</textarea> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> + +</body></html> diff --git a/doc/mmcount.html b/doc/mmcount.html new file mode 100644 index 00000000..1d06340d --- /dev/null +++ b/doc/mmcount.html @@ -0,0 +1,58 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>mmcount</title></head> + +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>mmcount</h1> +<p><b>Module Name: mmcount</b></p> +<p><b>Author: </b>Bala.FA <barumuga@redhat.com></p> +<p><b>Status: </b>Non project-supported module - contact author +or rsyslog mailing list for questions +<p><b>Available since</b>: 7.5.0</p> +<p><b>Description</b>:</p> +<p> +<pre> + mmcount: message modification plugin which counts messages + + This module provides the capability to count log messages by severity + or json property of given app-name. The count value is added into the + log message as json property named 'mmcount' + + Example usage of the module in the configuration file + + module(load="mmcount") + + # count each severity of appname gluster + action(type="mmcount" appname="gluster") + + # count each value of gf_code of appname gluster + action(type="mmcount" appname="glusterd" key="!gf_code") + + # count value 9999 of gf_code of appname gluster + action(type="mmcount" appname="glusterfsd" key="!gf_code" value="9999") + + # send email for every 50th mmcount + if $app-name == 'glusterfsd' and $!mmcount <> 0 and $!mmcount % 50 == 0 then { + $ActionMailSMTPServer smtp.example.com + $ActionMailFrom rsyslog@example.com + $ActionMailTo glusteradmin@example.com + $template mailSubject,"50th message of gf_code=9999 on %hostname%" + $template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'" + $ActionMailSubject mailSubject + $ActionExecOnlyOnceEveryInterval 30 + :ommail:;RSYSLOG_SyslogProtocol23Format + } +</pre> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> + +</body></html> diff --git a/doc/mmfields.html b/doc/mmfields.html new file mode 100644 index 00000000..885d6bca --- /dev/null +++ b/doc/mmfields.html @@ -0,0 +1,91 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>Field Extraction Module (mmfields)</title></head> + +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>Fields Extraction Module (mmfields)</h1> +<p><b>Module Name: mmfields</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available since</b>: 7.5.1</p> +<p><b>Description</b>:</p> +<p>The mmfield module permits to extract fields. It is an alternate to using +the property replacer field extraction capabilities. In contrast to the +property replacer, all fields are extracted as once and stored inside the +structured data part (more precisely: they become Lumberjack [JSON] properties). +<p>Using this module is of special advantage if a field-based log format +is to be processed, like for example CEF <b>and</b> and either a large +number of fields is needed or a specific field is used multiple times +inside filters. In these scenarios, mmfields potentially offers better +performance than the property replacer of the RainerScript field extraction +method. The reason is that mmfields extracts all fields as one big sweep, +whereas the other methods extract fields individually, which requires +multiple passes through the same data. On the other hand, adding field +content to the rsyslog property dictionary also has some overhead, +so for high-performance use cases it is suggested to do some performance +testing before finally deciding which method to use. This is most important +if only a smaller subset of the fields is actually needed. +<p>In any case, mmfields provides a very handy and easy to use way to +parse structured data into a it's individual data items. Again, a primiary +use case was support for CEF (Common Event Format), which is made +extremely easy to do with this module. +<p>This module is implemented via the action interface. Thus it +can be conditionally used depending on some prequisites. +<p> </p> + +<p><b>Module Configuration Parameters</b>:</p> +<p>Currently none. +<p> </p> +<p><b>Action Confguration Parameters</b>:</p> +<ul> +<li><b>separator</b> - separatorChar (default ',')<br> +This is the character used to separate fields. Currently, only a single +character is permitted, while the RainerScript method permits to +specify multi-character separator strings. For CEF, this is not required. +If there is actual need to support multi-character separator strings, +support can relatively easy be added. It is suggested to request it on the +rsyslog mailing list, together with the use case - we intend to add +functionality only if there is a real use case behind the request +(in the past we too-often implemented things that actually never got used). +<br>The fields are named f<i>nbr</i>, where <i>nbr</i> is the field number +starting with one and being incremented for each field. +<li><b>jsonRoot</b> - path (default "!")<br> +This parameters specifies into which json path the extracted fields shall +be written. The default is to use the json root object itself. +</ul> + +<p><b>Caveats/Known Bugs:</b> +<ul> +<li>Currently none. +</ul> + +<p><b>Samples:</b></p> +<p>This is a very simple use case where each message is +parsed. The default separator character of comma is being used. +<p><textarea rows="5" cols="60">module(load="mmfields") +template(name="ftpl" type=string string="%$!%\n") +action(type="omfields") +action(type="omfile" file="/path/to/logfile" template="ftpl") +</textarea> + +<p>The following sample is similar to the previous one, but +this time the colon is used as separator and data is written +into the "$!mmfields" json path. +<p><textarea rows="5" cols="60">module(load="mmfields") +template(name="ftpl" type=string string="%$!%\n") +action(type="omfields" separator=":" jsonRoot="!mmfields") +action(type="omfile" file="/path/to/logfile" template="ftpl") +</textarea> + + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> + +</body></html> diff --git a/doc/mmjsonparse.html b/doc/mmjsonparse.html new file mode 100644 index 00000000..c2c862d7 --- /dev/null +++ b/doc/mmjsonparse.html @@ -0,0 +1,45 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<title>CEE/lumberjack JSON support Module (mmjsonparse)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>Log Message Normalization Module</h1> +<p><b>Module Name: mmjsonparse</b></p> +<p><b>Available since: </b>6.6.0+ +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Description</b>:</p> +<p>This module provides support for parsing structured log messages +that follow the CEE/lumberjack spec. The so-called "CEE cookie" is checked +and, if present, the JSON-encoded structured message content is parsed. +The properties are than available as original message properties. +</p> +<p><b>Action specific Configuration Directives</b>:</p> +<p>currently none +<ul> +<p><b>Legacy Configuration Directives</b>:</p> +<p>none +<b>Caveats/Known Bugs:</b> +<p>None known at this time. +</ul> +<p><b>Sample:</b></p> +<p>This activates the module and applies normalization to all messages:<br> +</p> +<textarea rows="2" cols="60">module(load="mmjsonparse") +action(type="mmjsonparse") +</textarea> +<p>The same in legacy format:</p> +<textarea rows="2" cols="60">$ModLoad mmjsonparse +*.* :mmjsonparse: +</textarea> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2012 by <a href="http://www.gerhards.net/rainer">Rainer +Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/mmnormalize.html b/doc/mmnormalize.html index 82f9b6a2..911d6c89 100644 --- a/doc/mmnormalize.html +++ b/doc/mmnormalize.html @@ -11,37 +11,53 @@ <p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> <p><b>Description</b>:</p> <p>This module provides the capability to normalize log messages via -<a href="http://www.liblognorm.com">liblognorm</a>. Thanks to libee, unstructured text, +<a href="http://www.liblognorm.com">liblognorm</a>. Thanks to liblognorm, unstructured text, like usually found in log messages, can very quickly be parsed and put into -a normal form. This is done so quickly, that it usually should be possible +a normal form. This is done so quickly, that it should be possible to normalize events in realtime. -<p>This module is implemented via the output module interface. That means that +<p>This module is implemented via the output module interface. This means that mmnormalize should be called just like an action. After it has been called, -the normalized message properties are avaialable and can be access. These properties -are called the "CEE" properties, because liblognorm creates a format that is -inspired by the CEE approach. +the normalized message properties are available and can be accessed. These properties +are called the "CEE/lumberjack" properties, because liblognorm creates a format that is +inspired by the CEE/lumberjack approach. +<p><b>Please note:</b> CEE/lumberjack properties are different from regular properties. +They have always "$!" prepended to the property name given in the rulebase. Such a +property needs to be called with <b>%$!propertyname%</b>. <p>Note that mmnormalize should only be called once on each message. Behaviour is -undifined if multiple calls to mmnormalize happen for the same message. +undefined if multiple calls to mmnormalize happen for the same message. </p> -<p><b>Configuration Directives</b>:</p> +<p><b>Action Parameters</b>:</p> <ul> -<li>$mmnormalizeRuleBase <rulebase-file><br> -Specifies which rulebase file is to use. This file is loaded. If there are +<li><b>ruleBase</b> [word]<br> +Specifies which rulebase file is to use. If there are multiple mmnormalize instances, each one can use a different file. However, a single instance can use only a single file. This parameter MUST be given, -because normalization can only happen based on a rulebase. -<li>$mmnormalizeUseRawMsg <on/off><br> +because normalization can only happen based on a rulebase. It is recommended +that an absolute path name is given. Information on how to create the rulebase +can be found in the <a href="http://www.liblognorm.com/files/manual/index.html">liblognorm manual</a>. +<li><b>useRawMsg</b> [boolean]<br> Specifies if the raw message should be used for normalization (on) or just the MSG part of the message (off). Default is "off". </ul> +<p><b>Legacy Configuration Directives</b>:</p> +<ul> +<li>$mmnormalizeRuleBase <rulebase-file> - equivalent to the "ruleBase" +parameter. +<li>$mmnormalizeUseRawMsg <on/off> - equivalent to the "useRawMsg" +parameter. +</ul> <b>Caveats/Known Bugs:</b> <p>None known at this time. </ul> <p><b>Sample:</b></p> <p>This activates the module and applies normalization to all messages:<br> </p> -<textarea rows="8" cols="60">$ModLoad mmnormalize -$mmnormalizeRuleBase rulebase.rb +<textarea rows="2" cols="60">module(load="mmnormalize") +action(type="mmnormalize" ruleBase="/path/to/rulebase.rb") +</textarea> +<p>The same in legacy format:</p> +<textarea rows="3" cols="60">$ModLoad mmnormalize +$mmnormalizeRuleBase /path/to/rulebase.rb *.* :mmnormalize: </textarea> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] @@ -49,7 +65,7 @@ $mmnormalizeRuleBase rulebase.rb <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2010 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2010-2012 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/mmsnmptrapd.html b/doc/mmsnmptrapd.html index 699049d3..fb50f6c6 100644 --- a/doc/mmsnmptrapd.html +++ b/doc/mmsnmptrapd.html @@ -34,8 +34,8 @@ The following logic is applied to all message being processed: snmptrapd/severity/hostname. A configurable mapping table will be used to drive a new severity value from that severity string. If no mapping has been defined, the original severity is not changed. -<li>It replaces the "FromHost" value with the derived value from step2 -<li>It replaces the "Severity" value with the derived value from step 3 +<li>It replaces the "FromHost" value with the derived value from step 2 +<li>It replaces the "Severity" value with the derived value from step 3 </ol> <p>Note that the placement of this module inside the configuration is important. All actions before this modules is called will work on the unmodified message. All messages after it's call @@ -55,11 +55,11 @@ tells the module which start string inside the tag to look for. The default is matching incoming messages. It MUST not be given, except if two slashes are required for whatever reasons (so "tag/" results in a check for "tag//" at the start of the tag field). -<li><b>$mmsnmptrapdSeverityMapping</b> [severtiymap]<br> +<li><b>$mmsnmptrapdSeverityMapping</b> [severitymap]<br> This specifies the severity mapping table. It needs to be specified as a list. Note that due to the current config system <b>no whitespace</b> is supported inside the list, so be sure not to use any whitespace inside it.<br> -The list is constructed of Severtiy-Name/Severity-Value pairs, delimited by comma. +The list is constructed of Severity-Name/Severity-Value pairs, delimited by comma. Severity-Name is a case-sensitive string, e.g. "warning" and an associated numerical value (e.g. 4). Possible values are in the rage 0..7 and are defined in RFC5424, table 2. The diff --git a/doc/modules.html b/doc/modules.html index 4eae6db3..0ed7d4fe 100644 --- a/doc/modules.html +++ b/doc/modules.html @@ -70,7 +70,7 @@ other internal structures). Besides security, this also greatly simplifies the job of the output module developer.</p> <h2>Action Selectors</h2> <p>Modules (and rsyslog) need to know when they are called. For this, there must -a an action identification in selector lines. There are two syntaxes: the +be an action identification in selector lines. There are two syntaxes: the single-character syntax, where a single characters identifies a module (e.g. "*" for a wall message) and the modules designator syntax, where the module name is given between colons (e.g. ":ommysql:"). The single character syntax is diff --git a/doc/multi_ruleset.html b/doc/multi_ruleset.html index 37c54065..14a761c5 100644 --- a/doc/multi_ruleset.html +++ b/doc/multi_ruleset.html @@ -5,7 +5,7 @@ <h1>Multiple Rulesets in rsyslog</h1> <p>Starting with version 4.5.0 and 5.1.1, <a href="http://www.rsyslog.com">rsyslog</a> supports multiple rulesets within a single configuration. -This is especially useful for routing the recpetion of remote messages to a set of specific rules. +This is especially useful for routing the reception of remote messages to a set of specific rules. Note that the input module must support binding to non-standard rulesets, so the functionality may not be available with all inputs. <p>In this document, I am using <a href="imtcp.html">imtcp</a>, an input module @@ -39,7 +39,7 @@ is the name space reserved for rsyslog use). If it finds this directive, it begi rule set (if the name was not yet know) or switches to an already-existing one (if the name was known). All rules defined between this $RuleSet directive and the next one are appended to the named ruleset. Note that the reserved name "RSYSLOG_DefaultRuleset" is used to -specify rsyslogd's default ruleset. You can use that name whereever you can use a ruleset name, +specify rsyslogd's default ruleset. You can use that name wherever you can use a ruleset name, including when binding an input to it. <p>Inside a ruleset, messages are processed as described above: they start with the first rule @@ -48,7 +48,7 @@ there are no more rules or the discard action is executed. Note that with multip no longer <b>all</b> rsyslog.conf rules are executed but <b>only</b> those that are contained within the specific ruleset. -<p>Inputs must explicitely bind to rulesets. If they don't do, the default ruleset is bound. +<p>Inputs must explicitly bind to rulesets. If they don't do, the default ruleset is bound. <p>This brings up the next question: @@ -58,10 +58,10 @@ it means that a specific input, or part of an input (like a tcp listener) will u ruleset to "pass its messages to". So when a new message arrives, it will be processed via the bound ruleset. Rule from all other rulesets are irrelevant and will never be processed. <p>This makes multiple rulesets very handy to process local and remote message via -seperate means: bind the respective receivers to different rule sets, and you do not need -to seperate the messages by any other method. +separate means: bind the respective receivers to different rule sets, and you do not need +to separate the messages by any other method. -<p>Binding to rulesets is input-specifc. For imtcp, this is done via the +<p>Binding to rulesets is input-specific. For imtcp, this is done via the <pre>input(type="imptcp" port="514" ruleset="rulesetname"); </pre> @@ -72,7 +72,7 @@ I personally think that it is best to define all rule sets at the top of rsyslog define the inputs at the bottom. This kind of reverses the traditional recommended ordering, but seems to be a really useful and straightforward way of doing things. <h2>Why are rulesets important for different parser configurations?</h2> -<p>Custom message parsers, used to handle differnet (and potentially otherwise-invalid) +<p>Custom message parsers, used to handle different (and potentially otherwise-invalid) message formats, can be bound to rulesets. So multiple rulesets can be a very useful way to handle devices sending messages in different malformed formats in a consistent way. Unfortunately, this is not uncommon in the syslog world. An in-depth explanation @@ -191,7 +191,7 @@ ruleset(name="test1"){ } -# and now define listners bound to the relevant ruleset +# and now define listeners bound to the relevant ruleset input(type="imptcp" port="10514" ruleset="remote10514") input(type="imptcp" port="10515" ruleset="remote10515") input(type="imptcp" port="10516" ruleset="remote10516") @@ -222,7 +222,7 @@ needs to insert messages into the main message queue. So if each of them wants t submit a newly arrived message into the queue at the same time, only one can do so while the others need to wait. With multiple rulesets, its own queue can be created for each ruleset. If now each listener is bound to its own ruleset, concurrent message submission is -possible. On a machine with a sufficiently large number of corse, this can result in +possible. On a machine with a sufficiently large number of cores, this can result in dramatic performance improvement. <p>It is highly advised that high-performance systems define a dedicated ruleset, with a dedicated queue for each of the inputs. diff --git a/doc/multi_ruleset_legacy_format.html b/doc/multi_ruleset_legacy_format.html index 5a9e7a4a..03586ca7 100644 --- a/doc/multi_ruleset_legacy_format.html +++ b/doc/multi_ruleset_legacy_format.html @@ -5,18 +5,18 @@ <h1>Multiple Rulesets in rsyslog</h1> <p>Starting with version 4.5.0 and 5.1.1, <a href="http://www.rsyslog.com">rsyslog</a> supports multiple rulesets within a single configuration. -This is especially useful for routing the recpetion of remote messages to a set of specific rules. +This is especially useful for routing the reception of remote messages to a set of specific rules. Note that the input module must support binding to non-standard rulesets, so the functionality -may not be available with all inputs.<p> +may not be available with all inputs.</p> <b>Attention: this guide is shortened and only contains the samples in legacy format.</b> -Please follow this link to the full guide in the new config format "list": <a href="http://www.rsyslog.com/doc/multi_ruleset.html">http://www.rsyslog.com/doc/multi_ruleset.html<a> +Please follow this link to the full guide in the new config format "list": <a href="http://www.rsyslog.com/doc/multi_ruleset.html">http://www.rsyslog.com/doc/multi_ruleset.html</a> <h2>Examples</h2> <h3>Split local and remote logging</h3> <p>Let's say you have a pretty standard system that logs its local messages to the usual bunch of files that are specified in the default rsyslog.conf. As an example, your rsyslog.conf -might look like this: +might look like this:</p> <pre> # ... module loading ... @@ -34,7 +34,7 @@ cron.* /var/log/cron <p>Now, you want to add receive messages from a remote system and log these to a special file, but you do not want to have these messages written to the files specified above. The traditional approach is to add a rule in front of all others that -filters on the message, processes it and then discards it: +filters on the message, processes it and then discards it:</p> <pre> # ... module loading ... @@ -55,7 +55,7 @@ cron.* /var/log/cron </pre> <p>Note the tilde character, which is the discard action!. Also note that we assume that -192.0.2.1 is the sole remote sender (to keep it simple). +192.0.2.1 is the sole remote sender (to keep it simple).</p> <p>With multiple rulesets, we can simply define a dedicated ruleset for the remote reception case and bind it to the receiver. This may be written as follows: @@ -88,7 +88,7 @@ cron.* /var/log/cron <p>Here, we need to switch back to the default ruleset after we have defined our custom one. This is why I recommend a different ordering, which I find more intuitive. The sample -below has it, and it leads to the same results: +below has it, and it leads to the same results:</p> <pre> # ... module loading ... @@ -116,27 +116,27 @@ $InputTCPServerRun 10514 </pre> <p>Here, we do not switch back to the default ruleset, because this is not needed as it is -completely defined when we begin the "remote" ruleset. +completely defined when we begin the "remote" ruleset.</p> <p>Now look at the examples and compare them to the single-ruleset solution. You will notice that we do <b>not</b> need a real filter in the multi-ruleset case: we can simply use "*.*" as all messages now means all messages that are being processed by this rule set and all of them come in via the TCP receiver! This is what makes using multiple -rulesets so much easier. +rulesets so much easier.</p> <h3>Split local and remote logging for three different ports</h3> <p>This example is almost like the first one, but it extends it a little bit. While it is very similar, I hope it is different enough to provide a useful example why you may want -to have more than two rulesets. +to have more than two rulesets.</p> <p>Again, we would like to use the "regular" log files for local logging, only. But this time we set up three syslog/tcp listeners, each one listening to a different port (in this example 10514, 10515, and 10516). Logs received from these receivers shall go into different files. Also, logs received from 10516 (and only from that port!) with "mail.*" priority, shall be written into a specif file and <b>not</b> be -written to 10516's general log file. +written to 10516's general log file.</p> -<p>This is the config: +<p>This is the config:</p> <pre> # ... module loading ... @@ -168,7 +168,7 @@ mail.* /var/log/mail10516 # being written to the remote10516 file - as usual... *.* /var/log/remote10516 -# and now define listners bound to the relevant ruleset +# and now define listeners bound to the relevant ruleset $InputTCPServerBindRuleset remote10514 $InputTCPServerRun 10514 @@ -180,12 +180,12 @@ $InputTCPServerRun 10516 </pre> <p>Note that the "mail.*" rule inside the "remote10516" ruleset does -not affect processing inside any other rule set, including the default rule set. +not affect processing inside any other rule set, including the default rule set.</p> <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> -project.<br> +project.<br/> Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/ns_gtls.html b/doc/ns_gtls.html index 0d02ad02..21a7f19c 100644 --- a/doc/ns_gtls.html +++ b/doc/ns_gtls.html @@ -10,7 +10,7 @@ library</a>.</p> <p><b>Available since:</b> 3.19.0 (suggested minimum 3.19.8 and above)</p> <p style="font-weight: bold;">Supported Driver Modes</p> <ul> -<li>0 - unencrypted trasmission (just like <a href="ns_ptcp.html">ptcp</a> driver)</li> +<li>0 - unencrypted transmission (just like <a href="ns_ptcp.html">ptcp</a> driver)</li> <li>1 - TLS-protected operation</li> </ul> Note: mode 0 does not provide any benefit over the ptcp driver. This @@ -38,7 +38,7 @@ unauthorized access. It is recommended NOT to use this mode.</p> <p>x509/certvalid is a nonstandard mode. It validates the remote peers certificate, but does not check the subject name. This is weak authentication that may be useful in scenarios where multiple -devices are deployed and it is sufficient proof of authenticy when +devices are deployed and it is sufficient proof of authenticity when their certificates are signed by the CA the server trusts. This is better than anon authentication, but still not recommended. <b>Known Problems</b><br> diff --git a/doc/omelasticsearch.html b/doc/omelasticsearch.html new file mode 100644 index 00000000..618b7065 --- /dev/null +++ b/doc/omelasticsearch.html @@ -0,0 +1,177 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> + <head> + <meta content="en" http-equiv="Content-Language" /> + <title>Elasticsearch Output Module</title> + </head> + <body> + <p> + <a href="rsyslog_conf_modules.html">back</a></p> + <h1> + Elasticsearch Output Module</h1> + <p> + <b>Module Name: omelasticsearch</b></p> + <p> + <b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> + <p> + <b>Available since: </b>6.4.0+</p> + <p> + <b>Description</b>:</p> + <p> + This module provides native support for logging to <a href="http://www.elasticsearch.org/">Elasticsearch</a>.</p> + <p> + <b>Action Parameters</b>:</p> + <ul> + <li> + <b>server</b><br /> + Host name or IP address of the Elasticsearch server. Defaults to "localhost"</li> + <li> + <b>serverport</b><br /> + HTTP port to connect to Elasticsearch. Defaults to 9200</li> + <li> + <b>searchIndex</b><br /> + <a href="http://www.elasticsearch.org/guide/appendix/glossary.html#index">Elasticsearch index</a> to send your logs to. Defaults to "system"</li> + <li> + <b>dynSearchIndex </b><on/<b>off</b>><br /> + Whether the string provided for <strong>searchIndex</strong> should be taken as a <a href="http://www.rsyslog.com/doc/rsyslog_conf_templates.html">template</a>. Defaults to "off", which means the index name will be taken literally. Otherwise, it will look for a template with that name, and the resulting string will be the index name. For example, let's assume you define a template named "date-days" containing "%timereported:1:10:date-rfc3339%". Then, with dynSearchIndex="on", if you say searchIndex="date-days", each log will be sent to and index named after the first 10 characters of the timestamp, like "2013-03-22".</li> + <li> + <b>searchType</b><br /> + <a href="http://www.elasticsearch.org/guide/appendix/glossary.html#type">Elasticsearch type</a> to send your index to. Defaults to "events"</li> + <li> + <b>dynSearchType</b> <on/<strong>off</strong>><br /> + Like <strong>dynSearchIndex</strong>, it allows you to specify a <a href="http://www.rsyslog.com/doc/rsyslog_conf_templates.html">template</a> for <strong>searchType</strong>, instead of a static string.</li> + <li> + <strong>asyncrepl </strong><on/<strong>off</strong>><br /> + By default, an indexing operation returns after all <a href="http://www.elasticsearch.org/guide/appendix/glossary.html#replica_shard">replica shards</a> have indexed the document. With asyncrepl="on" it will return after it was indexed on the <a href="http://www.elasticsearch.org/guide/appendix/glossary.html#primary_shard">primary shard</a> only - thus trading some consistency for speed.</li> + <li> + <strong>timeout</strong><br /> + How long Elasticsearch will wait for a primary shard to be available for indexing your log before sending back an error. Defaults to "1m".</li> + <li> + <strong>template</strong><br /> + This is the JSON document that will be indexed in Elasticsearch. The resulting string needs to be a valid JSON, otherwise Elasticsearch will return an error. Defaults to:</li> + </ul> + <pre> +$template JSONDefault, "{\"message\":\"%msg:::json%\",\"fromhost\":\"%HOSTNAME:::json%\",\"facility\":\"%syslogfacility-text%\",\"priority\":\"%syslogpriority-text%\",\"timereported\":\"%timereported:::date-rfc3339%\",\"timegenerated\":\"%timegenerated:::date-rfc3339%\"}" +</pre> + <p> + Which will produce this sort of documents (pretty-printed here for readability):</p> + <ul> + </ul> + <pre> +{ + "message": " this is a test message", + "fromhost": "test-host", + "facility": "user", + "priority": "info", + "timereported": "2013-03-12T18:05:01.344864+02:00", + "timegenerated": "2013-03-12T18:05:01.344864+02:00" +}</pre> + <ul> + <li> + <strong>bulkmode </strong><on/<strong>off</strong>><br /> + The default "off" setting means logs are shipped one by one. Each in its own HTTP request, using the <a href="http://www.elasticsearch.org/guide/reference/api/index_.html">Index API</a>. Set it to "on" and it will use Elasticsearch's <a href="http://www.elasticsearch.org/guide/reference/api/bulk.html">Bulk API</a> to send multiple logs in the same request. The maximum number of logs sent in a single bulk request depends on your queue settings - usually limited by the <a href="http://www.rsyslog.com/doc/node35.html">dequeue batch size</a>. More information about queues can be found <a href="http://www.rsyslog.com/doc/node32.html">here</a>.</li> + <li> + <strong>parent</strong><br /> + Specifying a string here will index your logs with that string the parent ID of those logs. Please note that you need to define the <a href="http://www.elasticsearch.org/guide/reference/mapping/parent-field.html">parent field</a> in your <a href="http://www.elasticsearch.org/guide/reference/mapping/">mapping</a> for that to work. By default, logs are indexed without a parent.</li> + <li> + <strong>dynParent </strong><on/<strong>off</strong>><br /> + Using the same parent for all the logs sent in the same action is quite unlikely. So you'd probably want to turn this "on" and specify a <a href="http://www.rsyslog.com/doc/rsyslog_conf_templates.html">template</a> that will provide meaningful parent IDs for your logs.</li> + <li> + <strong>uid</strong><br /> + If you have basic HTTP authentication deployed (eg: through the <a href="https://github.com/Asquera/elasticsearch-http-basic">elasticsearch-basic plugin</a>), you can specify your user-name here.</li> + <li> + <strong>pwd</strong><br /> + Password for basic authentication.</li> + </ul> + <p> + <b>Samples:</b></p> + <p> + The following sample does the following:</p> + <ul> + <li> + loads the omelasticsearch module</li> + <li> + outputs all logs to Elasticsearch using the default settings</li> + </ul> + <pre> +module(load="omelasticsearch") +*.* action(type="omelasticsearch")</pre> + <p> + The following sample does the following:</p> + <ul> + <li> + loads the omelasticsearch module</li> + <li> + defines a template that will make the JSON contain the following properties (more info about what properties you can use <a href="http://www.rsyslog.com/doc/property_replacer.html">here</a>): + <ul> + <li> + RFC-3339 timestamp when the event was generated</li> + <li> + the message part of the event</li> + <li> + hostname of the system that generated the message</li> + <li> + severity of the event, as a string</li> + <li> + facility, as a string</li> + <li> + the tag of the event</li> + </ul> + </li> + <li> + outputs to Elasticsearch with the following settings + <ul> + <li> + host name of the server is myserver.local</li> + <li> + port is 9200</li> + <li> + JSON docs will look as defined in the template above</li> + <li> + index will be "test-index"</li> + <li> + type will be "test-type"</li> + <li> + activate bulk mode. For that to work effectively, we use an in-memory queue that can hold up to 5000 events. The maximum bulk size will be 300</li> + <li> + retry indefinitely if the HTTP request failed (eg: if the target server is down)</li> + </ul> + </li> + </ul> + <pre> +module(load="omelasticsearch") +template(name="testTemplate" + type="list" + option.json="on") { + constant(value="{") + constant(value="\"timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"message\":\"") property(name="msg") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"syslogtag\":\"") property(name="syslogtag") + constant(value="\"}") + } +*.* action(type="omelasticsearch" + server="myserver.local" + serverport="9200" + template="testTemplate" + searchIndex="test-index" + searchType="test-type" + bulkmode="on" + queue.type="linkedlist" + queue.size="5000" + queue.dequeuebatchsize="300" + action.resumeretrycount="-1")</pre> + <p> + </p> + <pre> +</pre> + <p> + [<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> + <p> + <font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br /> + Copyright © 2008-2012 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the ASL 2.0.</font></p> + </body> +</html> + diff --git a/doc/omfile.html b/doc/omfile.html index 23ecc034..72320921 100644 --- a/doc/omfile.html +++ b/doc/omfile.html @@ -13,14 +13,14 @@ <p>The omfile plug-in provides the core functionality of writing messages to files residing inside the local file system (which may actually be remote if methods like NFS are used). Both files named with static names as well files with names based on message content are supported by this module. It is a built-in module that does not need to be loaded. </p> <p> </p> -<p><b>Global Configuration Directives</b>:</p> +<p><b>Module Parameters</b>:</p> <ul> <li><strong>Template </strong>[templateName]<br> sets a new default template for file actions.<br></li> </ul> <p> </p> -<p><b>Action specific Configuration Directives</b>:</p> +<p><b>Action Parameters</b>:</p> <ul> <li><strong>DynaFileCacheSize </strong>(not mandatory, default will be used)<br> Defines a template to be used for the output. <br></li><br> @@ -28,6 +28,16 @@ <li><strong>ZipLevel </strong>0..9 [default 0]<br> if greater 0, turns on gzip compression of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.<br></li><br> + <li><b>VeryRobustZip</b> [<b>on</b>/off] (v7.3.0+) - if ZipLevel is greater 0, + then this setting controls if extra headers are written to make the resulting file + extra hardened against malfunction. If set to off, data appended to previously unclean + closed files may not be accessible without extra tools. + Note that this risk is usually expected to be bearable, and thus "off" is the default mode. + The extra headers considerably + degrade compression, files with this option set to "on" may be four to five times as + large as files processed in "off" mode. + </li><br> + <li><strong>FlushInterval </strong>(not mandatory, default will be used)<br> Defines a template to be used for the output. <br></li><br> @@ -47,10 +57,10 @@ Set the group for directories newly created. Please note that this setting does not affect the group of directories already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd on during startup processing. Interim changes to the user mapping are not detected.<br></li><br> <li><strong>FileOwner </strong><br> - Set the file owner for dynaFiles newly created. Please note that this setting does not affect the owner of files already existing. The parameter is a user name, for which the userid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.<br></li><br> + Set the file owner for files newly created. Please note that this setting does not affect the owner of files already existing. The parameter is a user name, for which the userid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.<br></li><br> <li><strong>FileGroup </strong><br> - Set the group for dynaFiles newly created. Please note that this setting does not affect the group of files already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.<br></li><br> + Set the group for files newly created. Please note that this setting does not affect the group of files already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.<br></li><br> <li><strong>DirCreateMode </strong>[defaul 0700]<br> This is the same as $FileCreateMode, but for directories automatically generated.<br></li><br> @@ -59,7 +69,7 @@ The FileCreateMode directive allows to specify the creation mode with which rsyslogd creates new files. If not specified, the value 0644 is used (which retains backward-compatibility with earlier releases). The value given must always be a 4-digit octal number, with the initial digit being zero. <br>Please note that the actual permission depend on rsyslogd's process umask. If in doubt, use "$umask 0000" right at the beginning of the configuration file to remove any restrictions. <br>FileCreateMode may be specified multiple times. If so, it specifies the creation mode for all selector lines that follow until the next $FileCreateMode directive. Order of lines is vitally important.<br></li><br> <li><strong>FailOnCHOwnFailure </strong>on/off [default on]<br> - This option modifies behaviour of dynaFile creation. If different owners or groups are specified for new files or directories and rsyslogd fails to set these new owners or groups, it will log an error and NOT write to the file in question if that option is set to "on". If it is set to "off", the error will be ignored and processing continues. Keep in mind, that the files in this case may be (in)accessible by people who should not have permission. The default is "on".<br></li><br> + This option modifies behaviour of file creation. If different owners or groups are specified for new files or directories and rsyslogd fails to set these new owners or groups, it will log an error and NOT write to the file in question if that option is set to "on". If it is set to "off", the error will be ignored and processing continues. Keep in mind, that the files in this case may be (in)accessible by people who should not have permission. The default is "on".<br></li><br> <li><strong>CreateDirs </strong>on/off [default on]<br> create directories on an as-needed basis<br></li><br> @@ -73,77 +83,106 @@ <li><strong>DynaFile </strong><br> For each message, the file name is generated based on the given template. Then, this file is opened. As with the ``file'' property, data is appended if the file already exists. If the file does not exist, a new file is created. A cache of recent files is kept. Note that this cache can consume quite some memory (especially if large buffer sizes are used). Files are kept open as long as they stay inside the cache. Currently, files are only evicted from the cache when there is need to do so (due to insufficient cache size). To force-close (and evict) a dynafile from cache, send a HUP signal to rsyslogd. <br></li><br> + <li><b>Sig.Provider </b>[ProviderName]<br> + Selects a signature provider for log signing. Currently, + there only is one provider called + "<a href="sigprov_gt.html">gt</a>".<br></li><br> + + <li><b>Cry.Provider </b>[ProviderName]<br> + Selects a crypto provider for log encryption. Currently, + there only is one provider called + "<a href="cryprov_gcry.html">gcry</a>".<br></li><br> + <li><strong>Template </strong>[templateName]<br> sets a new default template for file actions.<br></li><br> </ul> -<p><b>Caveats/Known Bugs:</b></p><ul><li>None.</li></ul> +<p><b>Caveats/Known Bugs:</b></p> +<ul> +<li>One needs to be careful with log rotation if signatures and/or encryption +are being used. These create side-files, which form a set and must be kept +together. +<br> +For signatures, the ".sigstate" file must NOT be rotated away if +signature chains are to be build across multiple files. This is because +.sigstate contains just global information for the whole file set. However, +all other files need to be rotated together. The proper sequence is to + <ol> + <li> move all files inside the file set + <li> only AFTER this is completely done, HUP rsyslog + </ol> +This sequence will ensure that all files inside the set are atomically +closed and in sync. HUPing only after a subset of files have been moved +results in inconsistencies and will most probably render the file set +unusable. +</li> +</ul> <p><b>Sample:</b></p> <p>The following command writes all syslog messages into a file.</p> -<textarea rows="5" cols="60">Module (path="builtin:omfile") +<textarea rows="5" cols="60">Module (load="builtin:omfile") *.* action(type="omfile" -DirCreateMode="0700" -FileCreateMode="0644" -File="/var/log/messages") + DirCreateMode="0700" + FileCreateMode="0644" + File="/var/log/messages") </textarea> <br><br> <p><b>Legacy Configuration Directives</b>:</p> <ul> - <li><strong>$DynaFileCacheSize </strong>(not mandatory, default will be used)<br> - Defines a template to be used for the output. <br></li><br> + <li><strong>$DynaFileCacheSize </strong><br> + equivalent to the "dynaFileCacheSize" parameter<br></li><br> - <li><strong>$OMFileZipLevel </strong>0..9 [default 0]<br> - if greater 0, turns on gzip compression of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.<br></li><br> + <li><strong>$OMFileZipLevel </strong><br> + equivalent to the "zipLevel" parameter<br></li><br> - <li><strong>$OMFileFlushInterval </strong>(not mandatory, default will be used)<br> - Defines a template to be used for the output. <br></li><br> + <li><strong>$OMFileFlushInterval </strong><br> + equivalent to the "flushInterval" parameter<br></li><br> - <li><strong>$OMFileASyncWriting </strong>on/off [default off]<br> - if turned on, the files will be written in asynchronous mode via a separate thread. In that case, double buffers will be used so that one buffer can be filled while the other buffer is being written. Note that in order to enable FlushInterval, AsyncWriting must be set to "on". Otherwise, the flush interval will be ignored. Also note that when FlushOnTXEnd is "on" but AsyncWriting is off, output will only be written when the buffer is full. This may take several hours, or even require a rsyslog shutdown. However, a buffer flush can be forced in that case by sending rsyslogd a HUP signal. <br></li><br> + <li><strong>$OMFileASyncWriting </strong><br> + equivalent to the "asyncWriting" parameter<br></li><br> - <li><strong>$OMFileFlushOnTXEnd </strong>on/off [default on]<br> - Omfile has the capability to write output using a buffered writer. Disk writes are only done when the buffer is full. So if an error happens during that write, data is potentially lost. In cases where this is unacceptable, set FlushOnTXEnd to on. Then, data is written at the end of each transaction (for pre-v5 this means after each log message) and the usual error recovery thus can handle write errors without data loss. Note that this option severely reduces the effect of zip compression and should be switched to off for that use case. Note that the default -on- is primarily an aid to preserve the traditional syslogd behaviour.<br></li><br> + <li><strong>$OMFileFlushOnTXEnd </strong><br> + equivalent to the "flushOnTXEnd" parameter<br></li><br> - <li><strong>$OMFileIOBufferSize </strong><size_nbr>, default 4k<br> - size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)<br></li><br> + <li><strong>$OMFileIOBufferSize </strong><br> + equivalent to the "IOBufferSize" parameter<br></li><br> <li><strong>$DirOwner </strong><br> - Set the file owner for directories newly created. Please note that this setting does not affect the owner of directories already existing. The parameter is a user name, for which the userid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.<br></li><br> + equivalent to the "dirOwner" parameter<br></li><br> <li><strong>$DirGroup </strong><br> - Set the group for directories newly created. Please note that this setting does not affect the group of directories already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd on during startup processing. Interim changes to the user mapping are not detected.<br></li><br> + equivalent to the "dirGroup" parameter<br></li><br> <li><strong>$FileOwner </strong><br> - Set the file owner for dynaFiles newly created. Please note that this setting does not affect the owner of files already existing. The parameter is a user name, for which the userid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.<br></li><br> + equivalent to the "fileOwner" parameter<br></li><br> <li><strong>$FileGroup </strong><br> - Set the group for dynaFiles newly created. Please note that this setting does not affect the group of files already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.<br></li><br> + equivalent to the "fileGroup" parameter<br></li><br> - <li><strong>$DirCreateMode </strong>[defaul 0700]<br> - This is the same as $FileCreateMode, but for directories automatically generated.<br></li><br> + <li><strong>$DirCreateMode </strong><br> + equivalent to the "dirCreateMode" parameter<br></li><br> - <li><strong>$FileCreateMode </strong>[default 0644]<br> - The FileCreateMode directive allows to specify the creation mode with which rsyslogd creates new files. If not specified, the value 0644 is used (which retains backward-compatibility with earlier releases). The value given must always be a 4-digit octal number, with the initial digit being zero. <br>Please note that the actual permission depend on rsyslogd's process umask. If in doubt, use "$umask 0000" right at the beginning of the configuration file to remove any restrictions. <br>FileCreateMode may be specified multiple times. If so, it specifies the creation mode for all selector lines that follow until the next $FileCreateMode directive. Order of lines is vitally important.<br></li><br> + <li><strong>$FileCreateMode </strong><br> + equivalent to the "fileCreateMode" parameter<br></li><br> - <li><strong>$FailOnCHOwnFailure </strong>on/off [default on]<br> - This option modifies behaviour of dynaFile creation. If different owners or groups are specified for new files or directories and rsyslogd fails to set these new owners or groups, it will log an error and NOT write to the file in question if that option is set to "on". If it is set to "off", the error will be ignored and processing continues. Keep in mind, that the files in this case may be (in)accessible by people who should not have permission. The default is "on".<br></li><br> + <li><strong>$FailOnCHOwnFailure </strong><br> + equivalent to the "failOnChOwnFailure" parameter<br></li><br> <li><strong>$F$OMFileForceCHOwn </strong><br> - force ownership change for all files<br></li><br> + equivalent to the "ForceChOwn" parameter<br></li><br> - <li><strong>$CreateDirs </strong>on/off [default on]<br> - create directories on an as-needed basis<br></li><br> + <li><strong>$CreateDirs </strong><br> + equivalent to the "createDirs" parameter<br></li><br> - <li><strong>$ActionFileEnableSync </strong>on/off [default off]<br> - enables file syncing capability of omfile.<br></li><br> + <li><strong>$ActionFileEnableSync </strong><br> + equivalent to the "enableSync" parameter<br></li><br> - <li><strong>$ActionFileDefaultTemplate </strong>[templateName]<br> - sets a new default template for file actions.<br></li><br> + <li><strong>$ActionFileDefaultTemplate </strong><br> + equivalent to the "template" module parameter<br></li><br> <li><strong>$ResetConfigVariables </strong><br> - Resets all configuration variables to their default value. Any settings made will not be applied to configuration lines following the $ResetConfigVariables. This is a good method to make sure no side-effects exists from previous directives. This directive has no parameters.<br></li><br> + Resets all configuration variables to their default value.<br></li><br> </ul> @@ -160,7 +199,7 @@ $FileCreateMode 0644 index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/omfwd.html b/doc/omfwd.html index 5599ae39..51aa58b5 100644 --- a/doc/omfwd.html +++ b/doc/omfwd.html @@ -16,7 +16,7 @@ <p><b>Global Configuration Directives</b>:</p> <ul> <li><strong>Template </strong>[templateName]<br> - sets a new default template for file actions.<br></li> + sets a non-standard default template for this module.<br></li> </ul> <p> </p> @@ -35,7 +35,38 @@ Framing-Mode to be for forwarding. This affects only TCP-based protocols. It is ignored for UDP. In protocol engineering, ``framing'' means how multiple messages over the same connection are separated. Usually, this is transparent to users. Unfortunately, the early syslog protocol evolved, and so there are cases where users need to specify the framing. The traditional framing is nontransparent. With it, messages are end when a LF (aka ``line break'', ``return'') is encountered, and the next message starts immediately after the LF. If multi-line messages are received, these are essentially broken up into multiple message, usually with all but the first message segment being incorrectly formatted. The octet-counting framing solves this issue. With it, each message is prefixed with the actual message length, so that a receivers knows exactly where the message ends. Multi-line messages cause no problem here. This mode is very close to the method described in RFC5425 for TLS-enabled syslog. Unfortunately, only few syslogd implementations support octet-counted framing. As such, the traditional framing is set as default, even though it has defects. If it is known that the receiver supports octet-counted framing, it is suggested to use that framing mode. <br></li><br> <li><strong>ZipLevel </strong>0..9 [default 0]<br> - Compression level for messages. Rsyslog implements a proprietary capability to zip transmitted messages. Note that compression happens on a message-per-message basis. As such, there is a performance gain only for larger messages. Before compressing a message, rsyslog checks if there is some gain by compression. If so, the message is sent compressed. If not, it is sent uncompressed. As such, it is totally valid that compressed and uncompressed messages are intermixed within a conversation. <br>The compression level is specified via the usual factor of 0 to 9, with 9 being the strongest compression (taking up most processing time) and 0 being no compression at all (taking up no extra processing time). <br></li><br> + Compression level for messages. + <br>Up until rsyslog 7.5.1, this was the only compression setting that + rsyslog understood. Starting with 7.5.1, we have different compression + modes. All of them are affected by the ziplevel. If, however, no mode + is explicitely set, setting ziplevel also turns on "single" + compression mode, so pre 7.5.1 configuration will continue to work + as expected. + <br>The compression level is specified via the usual factor of 0 to 9, with 9 being the strongest compression (taking up most processing time) and 0 being no compression at all (taking up no extra processing time). <br></li><br> + <li><b>compression.mode</b><i>mode</i><br> + <i>mode</i> is one of "none", "single", or "stream:always". The + default is "none", in which no compression happens at all. + <br>In "single" compression mode, Rsyslog implements a proprietary + capability to zip transmitted messages. That compression happens + on a message-per-message basis. As such, there is a performance gain + only for larger messages. Before compressing a message, rsyslog checks + if there is some gain by compression. If so, the message is sent + compressed. If not, it is sent uncompressed. As such, it is totally + valid that compressed and uncompressed messages are intermixed + within a conversation. + <br>In "stream:always" compression mode the full stream is being + compressed. This also uses non-standard protocol and is compatible + only with receives that have the same abilities. This mode offers + potentially very high compression ratios. With typical syslog + messages, it can be as high as 95+% compression (so only one twentieth + of data is actually transmitted!). Note that this mode introduces + extra latency, as data is only sent when the compressor emits new + compressed data. For typical syslog messages, this can mean that + some hundered messages may be held in local buffers before they are + actually sent. This mode has been introduced in 7.5.1. + <br><b>Note: currently only imptcp supports receiving stream-compressed + data.</b> + <br></li><br> <li><strong>RebindInterval </strong>integer<br> Permits to specify an interval at which the current connection is broken and re-established. This setting is primarily an aid to load balancers. After the configured number of messages has been transmitted, the current connection is terminated and a new one started. Note that this setting applies to both TCP and UDP traffic. For UDP, the new ``connection'' uses a different source port (ports are cycled and not reused too frequently). This usually is perceived as a ``new connection'' by load balancers, which in turn forward messages to another physical target system. <br></li><br> @@ -59,7 +90,7 @@ <p><b>Caveats/Known Bugs:</b></p><ul><li>None.</li></ul> <p><b>Sample:</b></p> <p>The following command sends all syslog messages to a remote server via TCP port 10514.</p> -<textarea rows="5" cols="60">Module (path="builtin:omfwd") +<textarea rows="5" cols="60">Module (load="builtin:omfwd") *.* action(type="omfwd" Target="192.168.2.11" Port="10514" diff --git a/doc/omjournal.html b/doc/omjournal.html new file mode 100644 index 00000000..6124e40c --- /dev/null +++ b/doc/omjournal.html @@ -0,0 +1,86 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>Linux Journal Output Module (omjournal)</title></head> + +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>Linux Journal Output Module (omjournal)</h1> +<p><b>Module Name: omjournal</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available since</b>: 7.3.7</p> +<p><b>Description</b>:</p> +<p>The omjournal output module provides an interface to the Linux journal. +It is meant to be used in those cases where the Linux journal is being used +as the sole system log database. With omjournal, messages from various +sources (e.g. files and remote devices) can also be written to the journal +and processed by its tools. +<p>A typical use case we had on our mind is a SOHO environment, where the +user wants to include syslog data obtained from the local router to be +part of the journal data. +<p>We suggest to check out our short presentation on +<a href="http://youtu.be/GTS7EuSdFKE">rsyslog journal integration</a> to +learn more details of anticipated use cases. +<p> </p> + +<p><b>Module Configuration Parameters</b>:</p> +<p>Currently none. +<p> </p> +<p><b>Action Confguration Parameters</b>:</p> +<p>Currently none. + +<p><b>Caveats/Known Bugs:</b> +<ul> +<li>One needs to be careful that no message routing loop is created. The +systemd journal forwards messages it receives to the traditional syslog +system (if present). That means rsyslog will receive the same message that +it just wrote as new input on imuxsock. If not handled specially and assuming +all messages be written to the journal, the message would be emitted to the +journal again and a deadly loop is started. +<p>To prevent that, imuxsock by default does not accept messages originating +from its own process ID, aka it ignores messages from the current instance of +rsyslogd. However, this setting can be changed, and if so the problem may occur. +</ul> + +<p><b>Sample:</b></p> +<p>We assume we have a DSL router inside the network and would like to +receive its syslog message into the journal. Note that this configuration can be +used without havoing any other syslog functionality at all (most importantly, there +is no need to write any file to /var/log!). We assume syslog over UDP, as this +is the most probable choice for the SOHO environment that this use case reflects. +To log to syslog data to the journal, add the following snippet to rsyslog.conf: +<textarea rows="20" cols="60">/* first, we make sure all necessary + * modules are present: + */ +module(load="imudp") # input module for UDP syslog +module(load="omjournal") # output module for journal + +/* then, define the actual server that listens to the + * router. Note that 514 is the default port for UDP + * syslog and that we use a dedicated ruleset to + * avoid mixing messages with the local log stream + * (if there is any). + */ +input(type="imudp" port="514" ruleset="writeToJournal") + +/* inside that ruleset, we just write data to the journal: */ +ruleset(name="writeToJournal") { + action(type="omjournal") +} +</textarea> +<p>Note that this can be your sole rsyslog.conf if you do not use rsyslog +for anything else than receving the router syslog messages. +<p>If you do not receive messages, <b>you probably need to enable inbound UDP +syslog traffic in your firewall</b>. + + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> + +</body></html> diff --git a/doc/omlibdbi.html b/doc/omlibdbi.html index 008dcb81..e47c7f57 100644 --- a/doc/omlibdbi.html +++ b/doc/omlibdbi.html @@ -54,7 +54,23 @@ dlopen()ed plugin (as omlibdbi is). So in short, you probably save you a lot of headache if you make sure you have at least libdbi version 0.8.3 on your system. </p> -<p><b>Action Parameters</b>:</p> +<p><b>Module Parameters</b></p> +<ul> +<li><b>template</b><br> +The default template to use. This template is used when no template is +explicitely specified in the action() statement. +<li><b>driverdirectory</b><br> +Path to the libdbi drivers. Usually, +you do not need to set it. If you installed libdbi-drivers at a +non-standard location, you may need to specify the directory here. If +you are unsure, do <b>not</b> use this configuration directive. +Usually, everything works just fine. +Note that this was an action() paramter in rsyslog versions below 7.3.0. +However, only the first action's driverdirectory parameter was actually used. +This has been cleaned up in 7.3.0, where this now is a module paramter. +</li> +</ul> +<p><b>Action Parameters</b></p> <ul> <li><b>server</b><br>Name or address of the MySQL server <li><b>db</b><br>Database to use @@ -68,24 +84,18 @@ writiting "mysql" (suggest to use ommysql instead), "firebird" (Firbird and InterBase), "ingres", "msql", "Oracle", "sqlite", "sqlite3", "freetds" (for Microsoft SQL and Sybase) and "pgsql" (suggest to use ompgsql instead).</li> -<li><b>driverdirectory</b><br> -Path to the libdbi drivers. Usually, -you do not need to set it. If you installed libdbi-drivers at a -non-standard location, you may need to specify the directory here. If -you are unsure, do <b>not</b> use this configuration directive. -Usually, everything works just fine.</li> </ul> <p><b>Legacy (pre-v6) Configuration Directives</b>:</p> +<p>It is strongly recommended NOT to use legacy format. <ul> -<li><b>$ActionLibdbiDriverDirectory /path/to/dbd/drivers</b> +<li><i>$ActionLibdbiDriverDirectory /path/to/dbd/drivers</i> - like the driverdirectory action parameter. -<li><strong>$ActionLibdbiDriver drivername</strong><br> - like the drivername action parameter. -<li><span style="font-weight: bold;">$ActionLibdbiHost hostname</span> - like the server action parameter -The host to connect to.</li> -<li><b>$ActionLibdbiUserName user</b> - like the uid action parameter -<li><b>$ActionlibdbiPassword</b> - like the pwd action parameter -<li><b>$ActionlibdbiDBName db</b> - like the db action parameter -<li><b>selector line: :omlibdbi:<i>;template</i></b><br> +<li><i>$ActionLibdbiDriver drivername</i> - like the drivername action parameter +<li><i>$ActionLibdbiHost hostname</i> - like the server action parameter +<li><i>$ActionLibdbiUserName user</i> - like the uid action parameter +<li><i>$ActionlibdbiPassword</i> - like the pwd action parameter +<li><i>$ActionlibdbiDBName db</i> - like the db action parameter +<li><i>selector line: :omlibdbi:<code>;template</code></i><br> executes the recently configured omlibdbi action. The ;template part is optional. If no template is provided, a default template is used (which is currently optimized for MySQL - sorry, folks...)</li> @@ -114,14 +124,14 @@ database "syslog_db" on mysqlsever.example.com. The server is MySQL and being accessed under the account of "user" with password "pwd" (if you have empty passwords, just remove the $ActionLibdbiPassword line).<br> </p> -<textarea rows="5" cols="60">$ModLoad omlibdbi +<textarea rows="5" cols="60">module(load="omlibdbi") *.* action(type="omlibdbi" driver="mysql" server="mysqlserver.example.com" db="syslog_db" uid="user" pwd="pwd" </textarea> -<p><b>Sample:</b></p> +<p><b>Legacy Sample:</b></p> <p>The same as above, but in legacy config format (pre rsyslog-v6): -<textarea rows="10" cols="60">$ModLoad omlibdbi +<textarea rows="8" cols="60">$ModLoad omlibdbi $ActionLibdbiDriver mysql $ActionLibdbiHost mysqlserver.example.com $ActionLibdbiUserName user diff --git a/doc/ommongodb.html b/doc/ommongodb.html new file mode 100644 index 00000000..a6112642 --- /dev/null +++ b/doc/ommongodb.html @@ -0,0 +1,60 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>MongoDB Output Module</title> +</head> + +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>MongoDB Output Module</h1> +<p><b>Module Name: ommongodb</b></p> +<p><b>Author: </b>Rainer Gerhards +<rgerhards@adiscon.com></p> +<p><b>Description</b>:</p> +<p>This module provides native support for logging to MongoDB. +</p> +<p><b>Action Parameters</b>:</p> +<ul> +<li><b>server</b><br>Name or address of the MongoDB server +<li><b>serverport</b><br>Permits to select +a non-standard port for the MongoDB server. The default is 0, which means the +system default port is used. There is no need to specify this parameter unless +you know the server is running on a non-standard listen port. +<li><b>db</b><br>Database to use +<li><b>collection</b><br>Collection to use +<li><b>uid</b><br>logon userid used to connect to server. Must have proper permissions. +<li><b>pwd</b><br>the user's password +<li><b>template</b><br>Template to use when submitting messages. +</ul> +<p>Note rsyslog contains a canned default template to write to the MongoDB. It +will be used automatically if no other template is specified to be used. This template is: +<p> +<textarea rows="5" cols="80">template(name="BSON" type="string" string="\"sys\" : \"%hostname%\", \"time\" : \"%timereported:::rfc3339%\", \"time_rcvd\" : \"%timegenerated:::rfc3339%\", \"msg\" : \"%msg%\", \"syslog_fac\" : \"%syslogfacility%\", \"syslog_sever\" : \"%syslogseverity%\", \"syslog_tag\" : \"%syslogtag%\", \"procid\" : \"%programname%\", \"pid\" : \"%procid%\", \"level\" : \"%syslogpriority-text%\"") +</textarea> +<p>This creates the BSON document needed for MongoDB if no template is specified. The default +schema is aligned to CEE and project lumberjack. As such, the field names are standard +lumberjack field names, and <b>not</b> +<a href="property_replacer.html">rsyslog property names</a>. When specifying templates, be sure +to use rsyslog property names as given in the table. If you would like to use lumberjack-based +field names inside MongoDB (which probably is useful depending on the use case), you need to +select fields names based on the lumberjack schema. +If you just want to use a subset of the fields, but with lumberjack names, you can look up the +mapping in the default template. For example, the lumberjack field "level" contains the rsyslog +property "syslogpriority-text". +<p><b>Sample:</b></p> +<p>The following sample writes all syslog messages to the +database "syslog" and into the collection "log" on mongosever.example.com. The server is +being accessed under the account of "user" with password "pwd". +</p> +<textarea rows="5" cols="80">module(load="ommongodb") +*.* action(type="ommongodb" server="mongoserver.example.com" db="syslog" collection="log" uid="user" pwd="pwd") +</textarea> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008-2012 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the ASL 2.0.</font></p> +</body></html> diff --git a/doc/omprog.html b/doc/omprog.html index 471ab224..4f369735 100644 --- a/doc/omprog.html +++ b/doc/omprog.html @@ -25,7 +25,34 @@ con re-using existing binaries. For the time being, it simply is not done. In th we may add an option for such pooling, provided that some demand for that is voiced. You can also mimic the same effect by defining multiple rulesets and including them (at the price of some slight performance loss). -<p><b>Configuration Directives</b>:</p> + +<p> </p> + +<p><b>Module Parameters</b>:</p> +<ul> + <li><strong>Template </strong>[templateName]<br> + sets a new default template for file actions.<br></li> + +</ul> +<p> </p> +<p><b>Action Parameters</b>:</p> +<ul> + <li><strong>binary </strong><br> + Mostly equivalent to the "binary" action parameter, but must contain the binary name + only. In legacy config, it is <b>not possible</b> to specify command line parameters. +</ul> +<p><b>Caveats/Known Bugs:</b></p><ul><li>None.</li></ul> +<p><b>Sample:</b></p> +<p>The following command writes all syslog messages into a file.</p> +<textarea rows="5" cols="85">Module (load="omprog") +*.* action(type="omprog" + binary="/pathto/omprog.py --parm1=\"value 1\" --parm2=value2" + template="RSYSLOG_TraditionalFileFormat") +</textarea> + +<br><br> + +<p><b>Legacy Configuration Directives</b>:</p> <ul> <li><b>$ActionOMProgBinary</b> <binary><br> The binary program to be executed. @@ -36,7 +63,7 @@ The binary program to be executed. [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/omrelp.html b/doc/omrelp.html index 22e6845f..8049ebaf 100644 --- a/doc/omrelp.html +++ b/doc/omrelp.html @@ -16,10 +16,107 @@ RELP protocol. For RELP's advantages over plain tcp syslog, please see the documentation for <a href="imrelp.html">imrelp</a> (the server counterpart). </p> <span style="font-weight: bold;">Setup</span> -<p>Please note the <a href="http://www.librelp.com">librelp</a> +<p>Please note that <a href="http://www.librelp.com">librelp</a> is required for imrelp (it provides the core relp protocol implementation).</p> -<p><b>Configuration Directives</b>:</p> +<p><b>Action Configuration Parameters</b>:</p> +<p>This module supports RainerScript configuration starting with +rsyslog 7.3.10. For older versions, legacy configuration directives +must be used. +<ul> + <li><b>target</b> (mandatory)<br> + The target server to connect to. + </li> + <li><b>template</b> (not mandatory, default "RSYSLOG_ForwardFormat")<br> + Defines the template to be used for the output. + </li> + <li><b>timeout</b> (not mandatory, default 90)<br> + Timeout for relp sessions. If set too low, valid sessions + may be considered dead and tried to recover. + </li> + <li><b>windowSize</b> (not mandatory, default 0)<br> + This is an <b>expert parameter</b>. It permits to override the + RELP window size being used by the client. Changing the window + size has both an effect on performance as well as potential + message duplication in failure case. A larger window size means + more performance, but also potentially more duplicated + messages - and vice versa. The default 0 means that librelp's + default window size is being used, which is considered a + compromise between goals reached. For your information: + at the time of this writing, the librelp default window size + is 128 messages, but this may change at any time. + <br>Note that there is no equivalent server parameter, as the + client proposes and manages the window size in RELP protocol. + <li><b>tls</b> (not mandatory, values "on","off", default "off")<br> + If set to "on", the RELP connection will be encrypted by TLS, so that the data is protected against observers. Please note that both the client and the server must have set TLS to either "on" or "off". Other combinations lead to unpredictable results. + </li> + <li><b>tls.compression</b> (not mandatory, values "on","off", default "off")<br> + The controls if the TLS stream should be compressed (zipped). While this + increases CPU use, the network bandwidth should be reduced. Note that + typical text-based log records usually compress rather well. + </li> + <li><b>tls.permittedPeer</b> peer</br> + Places access restrictions on this forwarder. Only peers which + have been listed in this parameter may be connected to. + This guards against rouge servers and man-in-the-middle + attacks. The validation + bases on the certficate the remote peer presents.<br> + The <i>peer</i> parameter lists permitted certificate + fingerprints. Note that it is an array parameter, so either + a single or multiple fingerprints can be listed. When a + non-permitted peer is connected to, the refusal is logged together + with it's fingerprint. So if the administrator knows this was + a valid request, he can simple add the fingerprint by copy and + paste from the logfile to rsyslog.conf. It must be noted, though, + that this situation should usually not happen after initial + client setup and administrators should be alert in this case. + <br>Note that usually a single remote peer should be all that + is ever needed. Support for multiple peers is primarily included + in support of load balancing scenarios. If the connection + goes to a specific server, only one specific certificate is ever + expected (just like when connecting to a specific ssh server). + <br>To specify multiple fingerprints, just enclose them + in braces like this: + <br>tls.permittedPeer=["SHA1:...1", "SHA1:....2"] + <br>To specify just a single peer, you can either + specify the string directly or enclose it in braces. + </li> + <li><b>tls.authMode</b> mode</br> + Sets the mode used for mutual authentication. Supported values are + either "<i>fingerprint</i>" or "<i>name"</i>. + <br>Fingerprint mode basically is what SSH + does. It does not require a full PKI to be present, instead self-signed + certs can be used on all peers. Even if a CA certificate is given, the + validity of the peer cert is NOT verified against it. Only the + certificate fingerprint counts. + <br>In "name" mode, certificate validation happens. Here, the matching + is done against the certificate's subjectAltName and, as a fallback, + the subject common name. If the certificate contains multiple names, + a match on any one of these names is considered good and permits the + peer to talk to rsyslog. + <li><b>tls.prioritystring</b> (not mandatory, string)<br> + This parameter permits to specify the so-called "priority string" to + GnuTLS. This string gives complete control over all crypto parameters, + including compression setting. For this reason, when the prioritystring + is specified, the "tls.compression" parameter has no effect and is + ignored. + <br>Full information about how to construct a priority string can be + found in the GnuTLS manual. At the time of this writing, this + information was contained in + <a href="http://gnutls.org/manual/html_node/Priority-Strings.html">section 6.10 of the GnuTLS manual</a>. + <br><b>Note: this is an expert parameter.</b> Do not use if you do + not exactly know what you are doing. + </li> +</ul> +<p><b>Sample:</b></p> +<p>The following sample sends all messages to the central server +"centralserv" at port 2514 (note that that server must run imrelp on +port 2514). +</p> +<textarea rows="3" cols="60">module(load="omrelp") +action(type="omrelp" target="centralserv" port="2514") +</textarea> +<p><b>Legacy Configuration Directives</b>:</p> <p>This module uses old-style action configuration to keep consistent with the forwarding rule. So far, no additional configuration directives can be specified. To send a message via RELP, @@ -33,18 +130,15 @@ use</p> <b>Caveats/Known Bugs:</b> <p>See <a href="imrelp.html">imrelp</a>, which documents them. </p> -<p><b>Sample:</b></p> +<p><b>Legacy Sample:</b></p> <p>The following sample sends all messages to the central server "centralserv" at port 2514 (note that that server must run imrelp on -port 2514). Rsyslog's high-precision timestamp format is used, thus the -special "RSYSLOG_ForwardFormat" (case sensitive!) template is used.<br> +port 2514). </p> -<textarea rows="15" cols="60">$ModLoad omrelp -# forward messages to the remote server "myserv" on -# port 2514 -*.* :omrelp:centralserv:2514;RSYSLOG_ForwardFormat +<textarea rows="3" cols="60">$ModLoad omrelp +*.* :omrelp:centralserv:2514 </textarea> -Note: to use IPv6 addresses, encode them in [::1] format. +<p>Note: to use IPv6 addresses, encode them in [::1] format. <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the diff --git a/doc/omudpspoof.html b/doc/omudpspoof.html index df14bbe1..930412c8 100644 --- a/doc/omudpspoof.html +++ b/doc/omudpspoof.html @@ -7,46 +7,161 @@ <h1>UDP spoofing output module (omudpspoof)</h1> <p><b>Module Name: omstdout</b></p> -<p><b>Author: </b>David Lang <david@lang.hm> and Rainer Gerhards -<rgerhards@adiscon.com></p> -<p><b>Available Since</b>: 5.1.3</p> +<p><b>Authors: </b>Rainer Gerhards <rgerhards@adiscon.com> +and David Lang <david@lang.hm> +</p> +<p><b>Available Since</b>: 5.1.3 / v7 config since 7.2.5</p> <p><b>Description</b>:</p> <p>This module is similar to the regular UDP forwarder, but permits to spoof the sender address. Also, it enables to circle through a number of source ports. -<p><b>Configuration Directives</b>:</p> +<p><b>Important:</b> This module requires root priveleges for its low-level +socket access. As such, the <b>module will not work if rsyslog is configured to +drop privileges</b>. + +<p><b>load() Parameters</b>:</p> +<ul> + <li><strong>Template </strong>[templateName]<br> + sets a non-standard default template for this module.<br></li> + +</ul> +<p> </p> +<p><b>action() parameters</b>:</p> +<ul> + <li><strong>Target </strong>string<br> + Name or IP-Address of the system that shall receive messages. Any resolvable name is fine. <br></li><br> + + <li><strong>Port </strong>[Integer, Default 514]<br> + Name or numerical value of port to use when connecting to target. <br></li><br> + + <li><b>Template</b>[Word]<br> + Template to use as message text. + <br></li><br> + + <li><strong>SourceTemplate </strong>[Word]<br> + This is the name of the template that contains a + numerical IP address that is to be used as the source system IP address. + While it may often be a constant value, it can be generated as usual via the + property replacer, as long as it is a valid IPv4 address. If not specified, the + build-in default template RSYSLOG_omudpspoofDfltSourceTpl is used. This template is defined + as follows:<br> + template(name="RSYSLOG_omudpspoofDfltSourceTpl" type="string" string="%fromhost-ip%")<br> + So in essence, the default template spoofs the address of the system the message + was received from. This is considered the most important use case. + <br></li><br> + + <li><b>SourcePortStart</b>[Word]<br> + Specifies the start value for circeling the source ports. Must be less than or + equal to the end value. Default is 32000. + <br></li><br> + + <li><b>SourcePortEnd</b>[Word]<br> + Specifies the ending value for circeling the source ports. Must be less than or + equal to the start value. Default is 42000. + <br></li><br> + + <li><b>mtu</b>[Integer, default 1500]<br> + Maximum MTU supported by the network. Default respects Ethernet and must + usually not be adjusted. Setting a too-high MTU can lead to message loss, + too low to excess message fragmentation. Change only if you really know what + you are doing. This is always given in number of bytes. + <br></li><br> +</ul> +<p><b>pre-v7 Configuration Directives</b>:</p> <ul> -<li><b>$ActionOMOMUDPSpoofSourceNameTemplate</b> <templatename><br> -This is the name of the template that contains a -numerical IP address that is to be used as the source system IP address. -While it may often be a constant value, it can be generated as usual via the -property replacer, as long as it is a valid IPv4 address. If not specified, the -build-in default template RSYSLOG_omudpspoofDfltSourceTpl is used. This template is defined -as follows:<br> -$template RSYSLOG_omudpspoofDfltSourceTpl,"%fromhost-ip%"<br> -So in essence, the default template spoofs the address of the system the message -was received from. This is considered the most important use case. -<li><b>$ActionOMUDPSpoofTargetHost</b> <hostname><br> -Host that the messages shall be sent to. -<li><b>$ActionOMUDPSpoofTargetPort</b> <port><br> -Remote port that the messages shall be sent to. -<li><b>$ActionOMUDPSpoofDefaultTemplate</b> <templatename><br> -This setting instructs omudpspoof to use a template different from the -default template for all of its actions that do not have a template specified -explicitely. -<li><b>$ActionOMUDPSpoofSourcePortStart</b> <number><br> -Specifies the start value for circeling the source ports. Must be less than or -equal to the end value. Default is 32000. -<li><b>$ActionOMUDPSpoofSourcePortEnd</b> <number><br> -Specifies the ending value for circeling the source ports. Must be less than or -equal to the start value. Default is 42000. +<li><b>$ActionOMOMUDPSpoofSourceNameTemplate</b> <templatename> +- equivalent to the "sourceTemplate" parameter. +<li><b>$ActionOMUDPSpoofTargetHost</b> <hostname> - equivalent to the "target" parameter. +<li><b>$ActionOMUDPSpoofTargetPort</b> <port> - equivalent to the "target" parameter. +<li><b>$ActionOMUDPSpoofDefaultTemplate</b> <templatename> +- equivalent to the "template" load() parameter. +<li><b>$ActionOMUDPSpoofSourcePortStart</b> <number> +- equivalent to the "SourcePortStart" parameter. +<li><b>$ActionOMUDPSpoofSourcePortEnd</b> <number> +- equivalent to the "SourcePortEnd" parameter. </ul> <b>Caveats/Known Bugs:</b> <ul> <li><b>IPv6</b> is currently not supported. If you need this capability, please let us know via the rsyslog mailing list. +<li>Versions shipped prior to rsyslog 7.2.5 do not support message sizes over 1472 bytes (more +pricesely: over the network-supported MTU). Starting with 7.2.5, those messages will be +fragmented, up to a total upper limit of 64K (induced by UDP). Message sizes over +64K will be truncated. For older versions, messages over 1472 may be totally discarded +or truncated, depending on version and environment. </ul> -<p><b>Sample:</b></p> + +<p><b>Config Samples</b></p> +<p>The following sample forwards all syslog messages in standard form to the +remote server server.example.com. The original sender's address is used. We do not +care about the source port. This example is considered the typical use case for +omudpspoof. +</p> +<textarea rows="3" cols="80">module(load="omudpspoof") +action(type="omudpspoof" target="server.example.com") +</textarea> + +<p>The following sample forwards all syslog messages in unmodified form to the +remote server server.example.com. The sender address 192.0.2.1 with fixed +source port 514 is used. +</p> +<textarea rows="7" cols="80">module(load="omudpspoof") +template(name="spoofaddr" type="string" string="192.0.2.1") +template(name="spooftemplate" type="string" string="%rawmsg%") +action(type="omudpspoof" target="server.example.com" + sourcetemplate="spoofaddr" template="spooftemplate" + sourceport.start="514" sourceport.end="514) +</textarea> +<p>The following sample is exatly like the previous, but it specifies a larger size +MTU. If, for example, the envrionment supports Jumbo Ethernet frames, increasing the +MTU is useful as it reduces packet fragmentation, which most often is the source of +problems. Note that setting the MTU to a value larger than the local-attached network +supports will lead to send errors and loss of message. So use with care! +</p> +<textarea rows="8" cols="80">module(load="omudpspoof") +template(name="spoofaddr" type="string" string="192.0.2.1") +template(name="spooftemplate" type="string" string="%rawmsg%") +action(type="omudpspoof" target="server.example.com" + sourcetemplate="spoofaddr" template="spooftemplate" + sourceport.start="514" sourceport.end="514 + mtu="8000") +</textarea> +<p>Of course, the action can be combined with any type of filter, for +example a tradition PRI filter:</p> +<textarea rows="8" cols="80">module(load="omudpspoof") +template(name="spoofaddr" type="string" string="192.0.2.1") +template(name="spooftemplate" type="string" string="%rawmsg%") +local0.* action(type="omudpspoof" target="server.example.com" + sourcetemplate="spoofaddr" template="spooftemplate" + sourceport.start="514" sourceport.end="514 + mtu="8000") +</textarea> +<p>... or any complex expression-based filter:</p> +<textarea rows="8" cols="80">module(load="omudpspoof") +template(name="spoofaddr" type="string" string="192.0.2.1") +template(name="spooftemplate" type="string" string="%rawmsg%") +if prifilt("local0.*") and $msg contains "error" then + action(type="omudpspoof" target="server.example.com" + sourcetemplate="spoofaddr" template="spooftemplate" + sourceport.start="514" sourceport.end="514 + mtu="8000") +</textarea> +<p>and of course it can also be combined with as many other actions +as one likes:</p> +<textarea rows="11" cols="80">module(load="omudpspoof") +template(name="spoofaddr" type="string" string="192.0.2.1") +template(name="spooftemplate" type="string" string="%rawmsg%") +if prifilt("local0.*") and $msg contains "error" then { + action(type="omudpspoof" target="server.example.com" + sourcetemplate="spoofaddr" template="spooftemplate" + sourceport.start="514" sourceport.end="514 + mtu="8000") + action(type="omfile" file="/var/log/somelog") + stop # or whatever... +} +</textarea> + +<p><b>Legacy Sample (pre-v7):</b></p> <p>The following sample forwards all syslog messages in standard form to the remote server server.example.com. The original sender's address is used. We do not care about the source port. This example is considered the typical use case for diff --git a/doc/property_replacer.html b/doc/property_replacer.html index c7624b2d..13ff41c3 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -413,6 +413,12 @@ option when forwarding to remote hosts - they may treat the date as invalid <td>just the subseconds of a timestamp (always 0 for a low precision timestamp)</td> </tr> <tr> +<td>pos-end-relative</td> + <td>the from and to position is relative to the end of the string + instead of the usual start of string. (available since rsyslog v7.3.10) + </td> +</tr> +<tr> <td><b>ControlCharacters</b></td> <td>Option values for how to process control characters</td> </tr> diff --git a/doc/rainerscript.html b/doc/rainerscript.html index 84261bdd..7cbbfa9f 100644 --- a/doc/rainerscript.html +++ b/doc/rainerscript.html @@ -51,6 +51,11 @@ of a and b should be tested as "a <> b". The "not" operator should be reserved to cases where it actually is needed to form a complex boolean expression. In those cases, parenthesis are highly recommended. +<h2>Lookup Tables</h2> +<p><a href="lookup_tables.html">Lookup tables</a> are a powerful construct +to obtain "class" information based on message content (e.g. to build +log file names for different server types, departments or remote +offices). <h2>Functions</h2> <p>RainerScript supports a currently quite limited set of functions: <ul> @@ -61,11 +66,30 @@ variable, if it exists. Returns an empty string if it does not exist. <li>cstr(expr) - converts expr to a string value <li>cnum(expr) - converts expr to a number (integer) <li>re_match(expr, re) - returns 1, if expr matches re, 0 otherwise +<li>re_extract(expr, re, match, submatch, no-found) - extracts +data from a string (property) via a regular expression match. +POSIX ERE regular expressions are used. The variable "match" contains +the number of the match to use. This permits to pick up more than the +first expression match. Submatch is the submatch to match (max 50 supported). +The "no-found" parameter specifies which string is to be returned in case when +the regular expression is not found. Note that match and submatch start with +zero. It currently is not possible to extract more than one submatch with +a single call. <li>field(str, delim, matchnbr) - returns a field-based substring. str is the string -to search, delim is the numerical ascii value of the field delimiter (so that -non-printable characters can by specified) and matchnbr is the match to search +to search, delim is the delimiter and matchnbr is the match to search for (the first match starts at 1). This works similar as the field based property-replacer option. +Versions prior to 7.3.7 only support a single character as delimiter character. +Starting with version 7.3.7, a full string can be used as delimiter. If a single +character is being used as delimiter, delim is the numerical ascii value of the +field delimiter character (so that non-printable characters can by specified). If a +string is used as delmiter, a multi-character string (e.g. "#011") is to be +specified. Samples:<br> +set $!usr!field = field($msg, 32, 3); -- the third field, delimited by space<br> +set $!usr!field = field($msg, "#011", 3); -- the third field, delmited by "#011"<br> +Note that when a single character is specified as string [field($msg, ",", 3)] a +string-based extraction is done, which is more performance intense than the +equivalent single-character [field($msg, 44 ,3)] extraction. <li>prifilt(constant) - mimics a traditional PRI-based filter (like "*.*" or "mail.info"). The traditional filter string must be given as a <b>constant string</b>. Dynamic string evaluation is not permitted (for performance reasons). @@ -80,7 +104,7 @@ if $msg contains getenv('TRIGGERVAR') then /path/to/errfile <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2012 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body></html> diff --git a/doc/rsconf1_rulesetcreatemainqueue.html b/doc/rsconf1_rulesetcreatemainqueue.html index 5c1e0dec..d09f95ce 100644 --- a/doc/rsconf1_rulesetcreatemainqueue.html +++ b/doc/rsconf1_rulesetcreatemainqueue.html @@ -58,7 +58,7 @@ mail.* /var/log/mail10516 # being written to the remote10516 file - as usual... *.* /var/log/remote10516 -# and now define listners bound to the relevant ruleset +# and now define listeners bound to the relevant ruleset $InputTCPServerBindRuleset remote10514 $InputTCPServerRun 10514 diff --git a/doc/rsconf1_rulesetparser.html b/doc/rsconf1_rulesetparser.html index ef29c2a8..433456c1 100644 --- a/doc/rsconf1_rulesetparser.html +++ b/doc/rsconf1_rulesetparser.html @@ -9,7 +9,7 @@ <p><b>Type:</b> ruleset-specific configuration directive</p> <p><b>Parameter Values:</b> string</p> <p><b>Available since:</b> 5.3.4+</p> -<p><b>Default:</b> rsyslog.rfc5424 followed by rsyslog.rfc5425</p> +<p><b>Default:</b> rsyslog.rfc5424 followed by rsyslog.rfc3164</p> <p><b>Description:</b></p> <p> This directive permits to specify which diff --git a/doc/rsyslog_conf_basic_structure.html b/doc/rsyslog_conf_basic_structure.html index 00a700d4..f5d4891a 100644 --- a/doc/rsyslog_conf_basic_structure.html +++ b/doc/rsyslog_conf_basic_structure.html @@ -80,7 +80,7 @@ A ruleset can be "bound" (assigned) to a specific input. In the analogy, this me a message comes in via that input, the "program" (ruleset) bound to it will be executed (but not any other!). <p>There is detail documentation available for -<a href="multi_ruleset">rsyslog rulesets</a>. +<a href="multi_ruleset.html">rsyslog rulesets</a>. <p>For quick reference, rulesets are defined as follows: <pre> ruleset(name="rulesetname") { diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 651808f6..e48ed6d3 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -17,12 +17,12 @@ appear as implementation progresses. many parameter settings modify queue parameters. If in doubt, use the default, it is usually well-chosen and applicable in most cases.</p> <ul> -<li><a href="rsconf1_abortonuncleanconfig.html">$AbortOnUncleanConfig</a> - abort startup if there is +<li><a href="rsconf1_abortonuncleanconfig.html"><b>$AbortOnUncleanConfig</b></a> - abort startup if there is any issue with the config file</li> <li><a href="rsconf1_actionexeconlywhenpreviousissuspended.html">$ActionExecOnlyWhenPreviousIsSuspended</a></li> -<li>$ActionName <a_single_word> - used primarily for documentation, e.g. when +<li><b>$ActionName</b> <a_single_word> - used primarily for documentation, e.g. when generating a configuration graph. Available sice 4.3.1. -<li>$ActionExecOnlyOnceEveryInterval <seconds> - +<li><b>$ActionExecOnlyOnceEveryInterval</b> <seconds> - execute action only if the last execute is at last <seconds> seconds in the past (more info in <a href="ommail.html">ommail</a>, but may be used with any action)</li> @@ -46,60 +46,60 @@ The question is if this is desired behavior? Or should the rule only be triggered if the messages occur within an e.g. 20 minute window? If the later is the case, you need a <br> -$ActionExecOnlyEveryNthTimeTimeout 1200 +<b>$ActionExecOnlyEveryNthTimeTimeout 1200</b> <br> This directive will timeout previous messages seen if they are older than 20 minutes. In the example above, the count would now be always 1 and consequently no rule would ever be triggered. -<li><a href="omfile.html">$ActionFileDefaultTemplate</a> [templateName] - sets a new default template for file actions</li> -<li><a href="omfile.html">$ActionFileEnableSync</a> [on/<span style="font-weight: bold;">off</span>] - enables file +<li><a href="omfile.html"><b>$ActionFileDefaultTemplate</b></a> [templateName] - sets a new default template for file actions</li> +<li><a href="omfile.html"><b>$ActionFileEnableSync</b></a> [on/<span style="font-weight: bold;">off</span>] - enables file syncing capability of omfile</li> -<li><a href="omfwd.html">$ActionForwardDefaultTemplate</a> [templateName] - sets a new +<li><a href="omfwd.html"><b>$ActionForwardDefaultTemplate</b></a> [templateName] - sets a new default template for UDP and plain TCP forwarding action</li> -<li>$ActionGSSForwardDefaultTemplate [templateName] - sets a +<li><b>$ActionGSSForwardDefaultTemplate</b> [templateName] - sets a new default template for GSS-API forwarding action</li> -<li>$ActionQueueCheckpointInterval <number></li> -<li>$ActionQueueDequeueBatchSize <number> [default 16]</li> -<li>$ActionQueueDequeueSlowdown <number> [number +<li><b>$ActionQueueCheckpointInterval</b> <number></li> +<li><b>$ActionQueueDequeueBatchSize</b> <number> [default 16]</li> +<li><b>$ActionQueueDequeueSlowdown</b> <number> [number is timeout in <i> micro</i>seconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]</li> -<li>$ActionQueueDiscardMark <number> [default +<li><b>$ActionQueueDiscardMark</b> <number> [default 9750]</li> -<li>$ActionQueueDiscardSeverity <number> +<li><b>$ActionQueueDiscardSeverity</b> <number> [*numerical* severity! default 8 (nothing discarded)]</li> -<li>$ActionQueueFileName <name></li> -<li>$ActionQueueHighWaterMark <number> [default +<li><b>$ActionQueueFileName</b> <name></li> +<li><b>$ActionQueueHighWaterMark</b> <number> [default 8000]</li> -<li>$ActionQueueImmediateShutdown [on/<b>off</b>]</li> -<li>$ActionQueueSize <number></li> -<li>$ActionQueueLowWaterMark <number> [default +<li><b>$ActionQueueImmediateShutdown</b> [on/<b>off</b>]</li> +<li><b>$ActionQueueSize</b> <number></li> +<li><b>$ActionQueueLowWaterMark</b> <number> [default 2000]</li> -<li>$ActionQueueMaxFileSize <size_nbr>, default 1m</li> -<li>$ActionQueueTimeoutActionCompletion <number> +<li><b>$ActionQueueMaxFileSize</b> <size_nbr>, default 1m</li> +<li><b>$ActionQueueTimeoutActionCompletion</b> <number> [number is timeout in ms (1000ms is 1sec!), default 1000, 0 means immediate!]</li> -<li>$ActionQueueTimeoutEnqueue <number> [number +<li><b>$ActionQueueTimeoutEnqueue</b> <number> [number is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li> -<li>$ActionQueueTimeoutShutdown <number> [number +<li><b>$ActionQueueTimeoutShutdown</b> <number> [number is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li> -<li>$ActionQueueWorkerTimeoutThreadShutdown +<li><b>$ActionQueueWorkerTimeoutThreadShutdown</b> <number> [number is timeout in ms (1000ms is 1sec!), default 60000 (1 minute)]</li> -<li>$ActionQueueType [FixedArray/LinkedList/<b>Direct</b>/Disk]</li> -<li>$ActionQueueSaveOnShutdown [on/<b>off</b>] +<li><b>$ActionQueueType</b> [FixedArray/LinkedList/<b>Direct</b>/Disk]</li> +<li><b>$ActionQueueSaveOnShutdown </b> [on/<b>off</b>] </li> -<li>$ActionQueueWorkerThreads <number>, num worker threads, default 1, recommended 1</li> -<li>$ActionQueueWorkerThreadMinumumMessages <number>, default 100</li> -<li><a href="rsconf1_actionresumeinterval.html">$ActionResumeInterval</a></li> -<li>$ActionResumeRetryCount <number> [default 0, -1 means eternal]</li> -<li><a href="omfwd.html">$ActionSendResendLastMsgOnReconnect</a> <[on/<b>off</b>]> specifies if the last message is to be resend when a connecition breaks and has been reconnected. May increase reliability, but comes at the risk of message duplication. -<li><a href="omfwd.html">$ActionSendStreamDriver</a> <driver basename> just like $DefaultNetstreamDriver, but for the specific action</li> -<li><a href="omfwd.html">$ActionSendStreamDriverMode</a> <mode>, default 0, mode to use with the stream driver (driver-specific)</li> -<li><a href="omfwd.html">$ActionSendStreamDriverAuthMode</a> <mode>, authentication mode to use with the stream driver. Note that this directive requires TLS +<li><b>$ActionQueueWorkerThreads</b> <number>, num worker threads, default 1, recommended 1</li> +<li><b>$ActionQueueWorkerThreadMinumumMessages</b> <number>, default 100</li> +<li><a href="rsconf1_actionresumeinterval.html"><b>$ActionResumeInterval</b></a></li> +<li><b>$ActionResumeRetryCount</b> <number> [default 0, -1 means eternal]</li> +<li><a href="omfwd.html"><b>$ActionSendResendLastMsgOnReconnect</b></a> <[on/<b>off</b>]> specifies if the last message is to be resend when a connecition breaks and has been reconnected. May increase reliability, but comes at the risk of message duplication. +<li><a href="omfwd.html"><b>$ActionSendStreamDriver</b></a> <driver basename> just like $DefaultNetstreamDriver, but for the specific action</li> +<li><a href="omfwd.html"><b>$ActionSendStreamDriverMode</b></a> <mode>, default 0, mode to use with the stream driver (driver-specific)</li> +<li><a href="omfwd.html"><b>$ActionSendStreamDriverAuthMode</b></a> <mode>, authentication mode to use with the stream driver. Note that this directive requires TLS netstream drivers. For all others, it will be ignored. (driver-specific)</li> -<li><a href="omfwd.html">$ActionSendStreamDriverPermittedPeer</a> <ID>, accepted fingerprint (SHA1) or name of remote peer. Note that this directive requires TLS +<li><a href="omfwd.html"><b>$ActionSendStreamDriverPermittedPeer</b></a> <ID>, accepted fingerprint (SHA1) or name of remote peer. Note that this directive requires TLS netstream drivers. For all others, it will be ignored. (driver-specific) -<span style="font-weight: bold;"> directive may go away</span>!</li> <li><a href="omfwd.html"><b>$ActionSendTCPRebindInterval</b> nbr</a>- [available since 4.5.1] - instructs the TCP send @@ -120,40 +120,40 @@ heartbeat. Note that this option auto-resets to "off", so if you inten actions, it must be specified in front off <b>all</b> selector lines that should provide this functionality. </li> -<li><a href="rsconf1_allowedsender.html">$AllowedSender</a></li> -<li><a href="rsconf1_controlcharacterescapeprefix.html">$ControlCharacterEscapePrefix</a></li> -<li><a href="rsconf1_debugprintcfsyslinehandlerlist.html">$DebugPrintCFSyslineHandlerList</a></li> +<li><a href="rsconf1_allowedsender.html"><b>$AllowedSender</b></a></li> +<li><a href="rsconf1_controlcharacterescapeprefix.html"><b>$ControlCharacterEscapePrefix</b></a></li> +<li><a href="rsconf1_debugprintcfsyslinehandlerlist.html"><b>$DebugPrintCFSyslineHandlerList</b></a></li> -<li><a href="rsconf1_debugprintmodulelist.html">$DebugPrintModuleList</a></li> -<li><a href="rsconf1_debugprinttemplatelist.html">$DebugPrintTemplateList</a></li> -<li>$DefaultNetstreamDriver <drivername>, the default <a href="netstream.html">network stream driver</a> to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem></li> -<li>$DefaultNetstreamDriverCertFile </path/to/certfile.pem></li> -<li>$DefaultNetstreamDriverKeyFile </path/to/keyfile.pem></li> +<li><a href="rsconf1_debugprintmodulelist.html"><b>$DebugPrintModuleList</b></a></li> +<li><a href="rsconf1_debugprinttemplatelist.html"><b>$DebugPrintTemplateList</b></a></li> +<li><b>$DefaultNetstreamDriver</b> <drivername>, the default <a href="netstream.html">network stream driver</a> to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem></li> +<li><b>$DefaultNetstreamDriverCertFile</b> </path/to/certfile.pem></li> +<li><b>$DefaultNetstreamDriverKeyFile</b> </path/to/keyfile.pem></li> <li><b>$DefaultRuleset</b> <i>name</i> - changes the default ruleset for unbound inputs to the provided <i>name</i> (the default default ruleset is named "RSYSLOG_DefaultRuleset"). It is advised to also read our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a>.</li> <li><a href="omfile.html"><b>$CreateDirs</b></a> [<b>on</b>/off] - create directories on an as-needed basis</li> -<li><a href="omfile.html">$DirCreateMode</a></li> -<li><a href="omfile.html">$DirGroup</a></li> -<li><a href="omfile.html">$DirOwner</a></li> -<li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li> -<li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li> -<li><a href="omfile.html">$DynaFileCacheSize</a></li> -<li><a href="rsconf1_escape8bitcharsonreceive.html">$Escape8BitCharactersOnReceive</a></li> -<li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> +<li><a href="omfile.html"><b>$DirCreateMode</b></a></li> +<li><a href="omfile.html"><b>$DirGroup</b></a></li> +<li><a href="omfile.html"><b>$DirOwner</b></a></li> +<li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html"><b>$DropMsgsWithMaliciousDnsPTRRecords</b></a></li> +<li><a href="rsconf1_droptrailinglfonreception.html"><b>$DropTrailingLFOnReception</b></a></li> +<li><a href="omfile.html"><b>$DynaFileCacheSize</b></a></li> +<li><a href="rsconf1_escape8bitcharsonreceive.html"><b>$Escape8BitCharactersOnReceive</b></a></li> +<li><a href="rsconf1_escapecontrolcharactersonreceive.html"><b>$EscapeControlCharactersOnReceive</b></a></li> <li><b>$EscapeControlCharactersOnReceive</b> [<b>on</b>|off] - escape USASCII HT character</li> -<li>$SpaceLFOnReceive [on/<b>off</b>] - instructs rsyslogd to replace LF with spaces during message reception (sysklogd compatibility aid)</li> -<li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> -<li><a href="omfile.html">$FailOnChownFailure</a></li> -<li><a href="omfile.html">$FileCreateMode</a></li> -<li><a href="omfile.html">$FileGroup</a></li> -<li><a href="omfile.html">$FileOwner</a></li> -<li><a href="rsconf1_generateconfiggraph.html">$GenerateConfigGraph</a></li> -<li><a href="rsconf1_gssforwardservicename.html">$GssForwardServiceName</a></li> -<li><a href="rsconf1_gsslistenservicename.html">$GssListenServiceName</a></li> -<li><a href="rsconf1_gssmode.html">$GssMode</a></li> -<li><a href="rsconf1_includeconfig.html">$IncludeConfig</a></li><li>MainMsgQueueCheckpointInterval <number></li> +<li><b>$SpaceLFOnReceive</b> [on/<b>off</b>] - instructs rsyslogd to replace LF with spaces during message reception (sysklogd compatibility aid)</li> +<li><b>$ErrorMessagesToStderr</b> [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> +<li><a href="omfile.html"><b>$FailOnChownFailure</b></a></li> +<li><a href="omfile.html"><b>$FileCreateMode</b></a></li> +<li><a href="omfile.html"><b>$FileGroup</b></a></li> +<li><a href="omfile.html"><b>$FileOwner</b></a></li> +<li><a href="rsconf1_generateconfiggraph.html"><b>$GenerateConfigGraph</b></a></li> +<li><a href="rsconf1_gssforwardservicename.html"><b>$GssForwardServiceName</b></a></li> +<li><a href="rsconf1_gsslistenservicename.html"><b>$GssListenServiceName</b></a></li> +<li><a href="rsconf1_gssmode.html"><b>$GssMode</b></a></li> +<li><a href="rsconf1_includeconfig.html"><b>$IncludeConfig</b></a></li><li>MainMsgQueueCheckpointInterval <number></li> <li><b>$LocalHostName</b> [name] - this directive permits to overwrite the system hostname with the one specified in the directive. If the directive is given multiple times, all but the last one will be ignored. Please note that startup @@ -166,39 +166,39 @@ This information might be needed by some log analyzers. If set to off, no such status messages are logged, what may be useful for other scenarios. [available since 4.7.0 and 5.3.0] <li><b>$MainMsgQueueDequeueBatchSize</b> <number> [default 32]</li> -<li>$MainMsgQueueDequeueSlowdown <number> [number +<li><b>$MainMsgQueueDequeueSlowdown</b> <number> [number is timeout in <i> micro</i>seconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]</li> -<li>$MainMsgQueueDiscardMark <number> [default 9750]</li> -<li>$MainMsgQueueDiscardSeverity <severity> +<li><b>$MainMsgQueueDiscardMark</b> <number> [default 9750]</li> +<li><b>$MainMsgQueueDiscardSeverity</b> <severity> [either a textual or numerical severity! default 4 (warning)]</li> -<li>$MainMsgQueueFileName <name></li> -<li>$MainMsgQueueHighWaterMark <number> [default +<li><b>$MainMsgQueueFileName</b> <name></li> +<li><b>$MainMsgQueueHighWaterMark</b> <number> [default 8000]</li> -<li>$MainMsgQueueImmediateShutdown [on/<b>off</b>]</li> -<li><a href="rsconf1_mainmsgqueuesize.html">$MainMsgQueueSize</a></li> -<li>$MainMsgQueueLowWaterMark <number> [default +<li><b>$MainMsgQueueImmediateShutdown</b> [on/<b>off</b>]</li> +<li><a href="rsconf1_mainmsgqueuesize.html"><b>$MainMsgQueueSize</b></a></li> +<li><b>$MainMsgQueueLowWaterMark</b> <number> [default 2000]</li> -<li>$MainMsgQueueMaxFileSize <size_nbr>, default +<li><b>$MainMsgQueueMaxFileSize</b> <size_nbr>, default 1m</li> -<li>$MainMsgQueueTimeoutActionCompletion +<li><b>$MainMsgQueueTimeoutActionCompletion</b> <number> [number is timeout in ms (1000ms is 1sec!), default 1000, 0 means immediate!]</li> -<li>$MainMsgQueueTimeoutEnqueue <number> [number +<li><b>$MainMsgQueueTimeoutEnqueue</b> <number> [number is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li> -<li>$MainMsgQueueTimeoutShutdown <number> [number +<li><b>$MainMsgQueueTimeoutShutdown</b> <number> [number is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li> -<li>$MainMsgQueueWorkerTimeoutThreadShutdown +<li><b>$MainMsgQueueWorkerTimeoutThreadShutdown</b> <number> [number is timeout in ms (1000ms is 1sec!), default 60000 (1 minute)]</li> -<li>$MainMsgQueueType [<b>FixedArray</b>/LinkedList/Direct/Disk]</li> -<li>$MainMsgQueueSaveOnShutdown [on/<b>off</b>] +<li><b>$MainMsgQueueType</b> [<b>FixedArray</b>/LinkedList/Direct/Disk]</li> +<li><b>$MainMsgQueueSaveOnShutdown </b> [on/<b>off</b>] </li> -<li>$MainMsgQueueWorkerThreads <number>, num +<li><b>$MainMsgQueueWorkerThreads</b> <number>, num worker threads, default 1, recommended 1</li> -<li>$MainMsgQueueWorkerThreadMinumumMessages <number>, default 100</li> -<li><a href="rsconf1_markmessageperiod.html">$MarkMessagePeriod</a> (immark)</li> +<li><b>$MainMsgQueueWorkerThreadMinumumMessages</b> <number>, default 100</li> +<li><a href="rsconf1_markmessageperiod.html"><b>$MarkMessagePeriod</b></a> (immark)</li> <li><b><i>$MaxMessageSize</i></b> <size_nbr>, default 2k - allows to specify maximum supported message size (both for sending and receiving). The default should be sufficient for almost all cases. Do not set this below 1k, as it would cause @@ -221,9 +221,9 @@ instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restri <br>Note that 2k, the current default, is the smallest size that must be supported in order to be compliant to the upcoming new syslog RFC series. </li> -<li><a href="rsconf1_maxopenfiles.html">$MaxOpenFiles</a></li> -<li><a href="rsconf1_moddir.html">$ModDir</a></li> -<li><a href="rsconf1_modload.html">$ModLoad</a></li> +<li><a href="rsconf1_maxopenfiles.html"><b>$MaxOpenFiles</b></a></li> +<li><a href="rsconf1_moddir.html"><b>$ModDir</b></a></li> +<li><a href="rsconf1_modload.html"><b>$ModLoad</b></a></li> <li><a href="omfile.html"><b>$OMFileAsyncWriting</b></a> [on/<b>off</b>], if turned on, the files will be written in asynchronous mode via a separate thread. In that case, double buffers will be used so that one buffer can be filled while the other buffer is being written. Note that in order @@ -246,15 +246,15 @@ error recovery thus can handle write errors without data loss. Note that this op severely reduces the effect of zip compression and should be switched to off for that use case. Note that the default -on- is primarily an aid to preserve the traditional syslogd behaviour.</li> -<li><a href="omfile.html">$omfileForceChown</a> - force ownership change for all files</li> +<li><a href="omfile.html"><b>$omfileForceChown</b></a> - force ownership change for all files</li> <li><b>$RepeatedMsgContainsOriginalMsg</b> [on/<b>off</b>] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most probably more (this may change from version to version, thus no specific limit is given). The bottom line is that n is large enough to get a good idea which message was repeated but it is not necessarily large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.</li> -<li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li> -<li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li> +<li><a href="rsconf1_repeatedmsgreduction.html"><b>$RepeatedMsgReduction</b></a></li> +<li><a href="rsconf1_resetconfigvariables.html"><b>$ResetConfigVariables</b></a></li> <li><b>$Ruleset</b> <i>name</i> - starts a new ruleset or switches back to one already defined. All following actions belong to that new rule set. the <i>name</i> does not yet exist, it is created. To switch back to rsyslog's @@ -268,17 +268,17 @@ a specific (list of) message parsers to be used with the ruleset. <li><b>$OptimizeForUniprocessor</b> [on/<b>off</b>] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common. [available since 4.1.0]</li> -<li>$PreserveFQDN [on/<b>off</b>) - if set to off (legacy default to remain compatible +<li><b>$PreserveFQDN</b> [on/<b>off</b>) - if set to off (legacy default to remain compatible to sysklogd), the domain part from a name that is within the same domain as the receiving system is stripped. If set to on, full names are always used.</li> -<li>$WorkDirectory <name> (directory for spool and other work files. +<li><b>$WorkDirectory</b> <name> (directory for spool and other work files. Do <b>not</b> use trailing slashes)</li> -<li>$UDPServerAddress <IP> (imudp) -- local IP +<li><b>$UDPServerAddress</b> <IP> (imudp) -- local IP address (or name) the UDP listens should bind to</li> -<li>$UDPServerRun <port> (imudp) -- former +<li><b>$UDPServerRun</b> <port> (imudp) -- former -r<port> option, default 514, start UDP server on this port, "*" means all addresses</li> -<li>$UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance +<li><b>$UDPServerTimeRequery</b> <nbr-of-times> (imudp) -- this is a performance optimization. Getting the system time is very costly. With this setting, imudp can be instructed to obtain the precise time only once every n-times. This logic is only activated if messages come in at a very fast rate, so doing less frequent @@ -286,10 +286,10 @@ time calls should usually be acceptable. The default value is two, because we ha seen that even without optimization the kernel often returns twice the identical time. You can set this value as high as you like, but do so at your own risk. The higher the value, the less precise the timestamp. -<li><a href="droppriv.html">$PrivDropToGroup</a></li> -<li><a href="droppriv.html">$PrivDropToGroupID</a></li> -<li><a href="droppriv.html">$PrivDropToUser</a></li> -<li><a href="droppriv.html">$PrivDropToUserID</a></li> +<li><a href="droppriv.html"><b>$PrivDropToGroup</b></a></li> +<li><a href="droppriv.html"><b>$PrivDropToGroupID</b></a></li> +<li><a href="droppriv.html"><b>$PrivDropToUser</b></a></li> +<li><a href="droppriv.html"><b>$PrivDropToUserID</b></a></li> <li><b>$Sleep</b> <seconds> - puts the rsyslog main thread to sleep for the specified number of seconds immediately when the directive is encountered. You should have a good reason for using this directive!</li> @@ -306,7 +306,7 @@ rsyslog.conf</b>. Otherwise, if error messages are triggered before this directi is processed, rsyslog will fix the local host IP to "127.0.0.1", what than can not be reset. </li> -<li><a href="rsconf1_umask.html">$UMASK</a></li> +<li><a href="rsconf1_umask.html"><b>$UMASK</b></a></li> </ul> <p><b>Where <size_nbr> or integers are specified above,</b> modifiers can be used after the number part. For example, 1k means diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 4186dac4..b6b0748b 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -45,6 +45,7 @@ to message generators. <li><a href="imsolaris.html">imsolaris</a> - input for the Sun Solaris system log source</li> <li><a href="im3195.html">im3195</a> - accepts syslog messages via RFC 3195</li> <li><a href="impstats.html">impstats</a> - provides periodic statistics of rsyslog internal counters</li> +<li><a href="imjournal.html">imjournal</a> - Linux journal inuput module</li> </ul> <a name"om"></a><h2>Output Modules</h2> @@ -53,6 +54,7 @@ and messages be transmitted to various different targets. <ul> <li><a href="omfile.html">omfile</a> - file output module</li> <li><a href="omfwd.html">omfwd</a> - syslog forwarding output module</li> +<li><a href="omjournal.html">omjournal</a> - Linux journal output module</li> <li><a href="ompipe.html">ompipe</a> - named pipe output module</li> <li><a href="omusrmsg.html">omusrmsg</a> - user message output module</li> <li><a href="omsnmp.html">omsnmp</a> - SNMP trap output module</li> @@ -72,6 +74,8 @@ permits rsyslog to alert folks by mail if something important happens</li> <li><a href="omudpspoof.html">omudpspoof</a> - output module sending UDP syslog messages with a spoofed address</li> <li><a href="omuxsock.html">omuxsock</a> - output module Unix domain sockets</li> <li><a href="omhdfs.html">omhdfs</a> - output module for Hadoop's HDFS file system</li> +<li><a href="ommongodb.html">ommongodb</a> - output module for MongoDB</li> +<li><a href="omelasticsearch.html">omelasticsearch</a> - output module for ElasticSearch</li> </ul> <a name="pm"></a><h2>Parser Modules</h2> @@ -99,18 +103,21 @@ They can be implemented using either the output module or the parser module inte From the rsyslog core's point of view, they actually are output or parser modules, it is their implementation that makes them special. <p>Currently, there exists only a limited set of such modules, but new ones could be written with -the methods the engine provides. They could be used, for example, to: -<ul> -<li>anonymize message content -<li>add dynamically computed content to message (fields) -</ul> +the methods the engine provides. They could be used, for example, to +add dynamically computed content to message (fields). <p>Message modification modules are usually written for one specific task and thus usually are not generic enough to be reused. However, existing module's code is probably an excellent starting base for writing a new module. Currently, the following modules exist inside the source tree: <ul> +<li><a href="mmanon.html">mmanon</a> - used to anonymize log messages. +<li><a href="mmcount.html">mmcount</a> - message modification plugin which counts messages +<li><a href="mmfields.html">mmfields</a> - used to extract fields from +specially formatted messages (e.g. CEF) <li><a href="mmnormalize.html">mmnormalize</a> - used to normalize log messages. Note that this actually is a <b>generic</b> module. +<li><a href="mmjsonparse.html">mmjsonparse</a> - used to interpret CEE/lumberjack +enabled structured log messages. <li><a href="mmsnmptrapd.html">mmsnmptrapd</a> - uses information provided by snmptrapd inside the tag to correct the original sender system and priority of messages. Implemented via the output module interface. @@ -179,7 +186,7 @@ filter settings. This graphic above is a high-level message flow diagram. [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body> diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index 0c189100..9a6e1619 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -134,6 +134,8 @@ csv-data is generated, "json", which formats proper json content (but without a header) and "jsonf", which formats as a complete json field. <li>position.from - obtain substring starting from this position (1 is the first position) <li>position.to - obtain substring up to this position +<li>position.relativeToEnd - the from and to position is relative to the end of the string + instead of the usual start of string. (available since rsyslog v7.3.10) <li>field.number - obtain this field match <li>field.delimiter - decimal value of delimiter character for field extraction <li>regex.expression - expression to use diff --git a/doc/sigprov_gt.html b/doc/sigprov_gt.html new file mode 100644 index 00000000..caeee116 --- /dev/null +++ b/doc/sigprov_gt.html @@ -0,0 +1,100 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>GuardTime Log Signature Provider (gt)</title> +</head> + +<body> +<a href="rsyslog_conf_modules.html">back to rsyslog module overview</a> + +<h1>GuardTime Log Signature Provider (gt)</h1> +<p><b>Signature Provider Name: gt</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Supported Since: </b>since 7.3.9 +<p><b>Description</b>:</p> +<p>Provides the ability to sign syslog messages via the +GuardTime signature services. +</p> + +<p><b>Configuration Parameters</b>:</p> +<p>Signature providers are loaded by omfile, when the +provider is selected in its "sig.providerName" parameter. +Parameters for the provider are given in the omfile action instance +line. +<p>This provider creates a signature file with the same base name but +the extension ".gtsig" for each log file (both for fixed-name files +as well as dynafiles). Both files together form a set. So you need to +archive both in order to prove integrity. +<ul> +<li><b>sig.hashFunction</b> <Hash Algorithm><br> +The following hash algorithms are currently supported: + <ul> + <li>SHA1 + <li>RIPEMD-160 + <li>SHA2-224 + <li>SHA2-256 + <li>SHA2-384 + <li>SHA2-512 + </ul> +</li> +<li><b>sig.timestampService</b> <timestamper URL><br> +This provides the URL of the timestamper service. If not selected, +a default server is selected. This may not necessarily be a good +one for your region. +</li> +<li><b>sig.block.sizeLimit</b> <nbr-records><br> +The maximum number of records inside a single signature block. By +default, there is no size limit, so the signature is only written +on file closure. Note that a signature request typically takes between +one and two seconds. So signing to frequently is probably not a good +idea. +</li> +<li><b>sig.keepRecordHashes</b> <on/<b>off</b>><br> +Controls if record hashes are written to the .gtsig file. This +enhances the ability to spot the location of a signature breach, +but costs considerable disk space (65 bytes for each log record +for SHA2-512 hashes, for example). +</li> +<li><b>sig.keepTreeHashes</b> <on/<b>off</b>><br> +Controls if tree (intermediate) hashes are written to the .gtsig file. This +enhances the ability to spot the location of a signature breach, +but costs considerable disk space (a bit mire than the amount +sig.keepRecordHashes requries). Note that both Tree and Record +hashes can be kept inside the signature file. +</li> +</ul> +<b>Caveats/Known Bugs:</b> +<ul> +<li>currently none known +</li> +</ul> +<p><b>Samples:</b></p> +<p>This writes a log file with it's associated signature file. Default +parameters are used. +</p> +<textarea rows="3" cols="60"> +action(type="omfile" file="/var/log/somelog" + sig.provider="gt") +</textarea> + +<p>In the next sample, we use the more secure SHA2-512 hash function, +sign every 10,000 records and Tree and Record hashes are kept. +<textarea rows="3" cols="60"> +action(type="omfile" file="/var/log/somelog" + sig.provider="gt" sig.hashfunction="SHA2-512" + sig.block.sizelimit="10000" + sig.keepTreeHashes="on" sig.keepRecordHashes="on") +</textarea> + + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2013 by +<a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/v7compatibility.html b/doc/v7compatibility.html index 01faacac..da4772fe 100644 --- a/doc/v7compatibility.html +++ b/doc/v7compatibility.html @@ -82,6 +82,53 @@ errors and error messages. Starting with 7.2.1, this has been reduced to 10 successive failures. This still gives the plugin a chance to recover. In extreme cases, a plugin may now enter suspend mode where it previously did not do so. In practice, we do NOT expect that. +<h1>Notes for the 7.3/7.4 branch</h1> +<h2>"last message repeated n times" Processing</h2> +<p>This processing has been optimized and moved to the input side. This results +in usually far better performance and also de-couples different sources +from the same +processing. It is now also integrated in to the more generic rate-limiting +processing. +<h3>User-Noticable Changes</h3> +The code works almost as before, with two exceptions: +<ul> +<li>The supression amount can be different, as the new algorithm + precisely check's a single source, and while that source is being + read. The previous algorithm worked on a set of mixed messages + from multiple sources. +<li>The previous algorithm wrote a "last message repeated n times" message + at least every 60 seconds. For performance reasons, we do no longer do + this but write this message only when a new message arrives or rsyslog + is shut down. +</ul> +<p>Note that the new algorithms needs support from input modules. If old +modules which do not have the necessary support are used, duplicate +messages will most probably not be detected. Upgrading the module code is +simple, and all rsyslog-provided plugins support the new method, so this +should not be a real problem (crafting a solution would result in rather +complex code - for a case that most probably would never happen). +<h3>Performance Implications</h3> +<p>In general, the new method enables far faster output procesing. However, it +needs to be noted that the "last message repeated n" processing needs parsed +messages in order to detect duplicated. Consequently, if it is enabled the +parser step cannot be deferred to the main queue processing thread and +thus must be done during input processing. The changes workload distribution +and may have (good or bad) effect on the overall performance. If you have +a very high performance installation, it is suggested to check the performance +profile before deploying the new version. Note: for high-performance +environments it is highly recommended NOT to use "last message repeated n times" +processing but rather the other (more efficient) rate-limiting methods. These +also do NOT require the parsing step to be done during input processing. + +<h2>Stricter string-template Processing</h2> +<p>Previously, no error message for invalid string template parameters +was generated. +Rather a malformed template was generated, and error information emitted +at runtime. However, this could be quite confusing. Note that the new code +changes user experience: formerly, rsyslog and the affected +actions properly started up, but the actions did not produce proper +data. Now, there are startup error messages and the actions are NOT +executed (due to missing template due to template error). <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> |