diff options
384 files changed, 23973 insertions, 8016 deletions
@@ -1,19 +1,14 @@ -Rainer Gerhards <rgerhards@adiscon.com>, Adiscon GmbH -Michael Meckelein <mmeckelein@hq.adiscon.com>, Adiscon GmbH +Thankfully, we have had so many contributions that maintaining the +AUTHORS file would be a big task in itself. On the other hand, we +now use git and I make sure that each author receives proper credit +for patches I receive. -Contributors -Michael Biebl - - helped continously with autotools - - provided numerous advise on how to do things under Linux - - provided excellent advise in many, many cases -Andres Riancho (andres-dot-riancho-at-gmail-dot-com) - (alias APR in code files) - - supplied regexp functionality for the property replacer - a great feature. - thanks! -Bjoern Kalkbrenner - - provided code for the "execute shell script" action -Peter Vrabec - - provided IPv6-enabling code -varmojfekoj - - helped with a variety of things and, most importantly, contributed - the gssapi functionality +So rather than trying to reproduce the git author log here (and +often making mistakes in that), I invite you to check the git logs. +You can also do this online at + +http://git.adiscon.com/?p=rsyslog.git;a=summary + +Rainer Gerhards +<rgerhards@adiscon.com> +lead rsyslog developer @@ -1,6 +1,714 @@ --------------------------------------------------------------------------- -Version 4.3.? [DEVEL] (rgerhards), 2009-??-?? +Version 5.5.3 [DEVEL] (rgerhards), 2010-02-?? +- added capability to turn off standard LF delimiter in TCP server + via new directive "$InputTCPServerDisableLFDelimiter on" +- bugfix: failed to compile on systems without epoll support +- bugfix: comment char ('#') in literal terminated script parsing + and thus could not be used. + but tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=119 + [merged in from v3.22.2] +- imported patches from 4.6.0: + * improved testbench to contain samples for totally malformed messages + which miss parts of the message content + * bugfix: some malformed messages could lead to a missing LF inside files + or some other missing parts of the template content. + * bugfix: if a message ended immediately with a hostname, the hostname + was mistakenly interpreted as TAG, and localhost be used as hostname +--------------------------------------------------------------------------- +Version 5.5.2 [DEVEL] (rgerhards), 2010-02-05 +- applied patches that make rsyslog compile under Apple OS X. + Thanks to trey for providing these. +- replaced data type "bool" by "sbool" because this created some + portability issues. +- added $Escape8BitCharactersOnReceive directive + Thanks to David Lang for suggesting it. +- worked around an issue where omfile failed to compile on 32 bit platforms + under some circumstances (this smells like a gcc problem, but a simple + solution was available). Thanks to Kenneth Marshall for some advice. +- extended testbench +--------------------------------------------------------------------------- +Version 5.3.8 [BETA] (rgerhards), 2010-02-?? +- bugfix: omruleset failed to work in many cases + bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=179 + Thanks to Ryan B. Lynch for reporting this issue. +- bugfix: comment char ('#') in literal terminated script parsing + and thus could not be used. + but tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=119 + [merged in from v3.22.2] +--------------------------------------------------------------------------- +Version 5.3.7 [BETA] (rgerhards), 2010-01-27 +- bugfix: queues in direct mode could case a segfault, especially if an + action failed for action queues. The issue was an invalid increment of + a stack-based pointer which lead to destruction of the stack frame and + thus a segfault on function return. + Thanks to Michael Biebl for alerting us on this problem. +- bugfix: hostname accidently set to IP address for some message sources, + for example imudp. Thanks to Anton for reporting this bug. [imported v4] +- bugfix: ompgsql had problems with transaction support, what actually + rendered it unsuable. Thanks to forum user "horhe" for alerting me + on this bug and helping to debug/fix it! [imported from 5.3.6] +- bugfix: $CreateDirs variable not properly initialized, default thus + was random (but most often "on") [imported from v3] +--------------------------------------------------------------------------- +Version 5.5.1 [DEVEL] (rgerhards), 2009-11-27 +- introduced the ablity for netstream drivers to utilize an epoll interface + This offers increased performance and removes the select() FDSET size + limit from imtcp. Note that we fall back to select() if there is no + epoll netstream drivers. So far, an epoll driver has only been + implemented for plain tcp syslog, the rest will follow once the code + proves well in practice AND there is demand. +- re-implemented $EscapeControlCharacterTab config directive + Based on Jonathan Bond-Caron's patch for v4. This now also includes some + automatted tests. +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +--------------------------------------------------------------------------- +Version 5.5.0 [DEVEL] (rgerhards), 2009-11-18 +- moved DNS resolution code out of imudp and into the backend processing + Most importantly, DNS resolution now never happens if the resolved name + is not required. Note that this applies to imudp - for the other inputs, + DNS resolution almost comes for free, so we do not do it there. However, + the new method has been implemented in a generic way and as such may + also be used by other modules in the future. +- added option to use unlimited-size select() calls + Thanks to varmjofekoj for the patch + This is not done in imudp, as it natively supports epoll(). +- doc: improved description of what loadable modules can do +--------------------------------------------------------------------------- +Version 5.3.7 [BETA] (rgerhards), 2010-01-?? +- bugfix: potential segfaults during queue shutdown + (bugs require certain non-standard settings to appear) + Thanks to varmojfekoj for the patch [imported from 4.5.8] + [backport from 5.5.2] +- bugfix: wrong memory assignment for a config variable (probably + without causing any harm) [backport from 5.2.2] +- bugfix: rsyslog hangs when writing to a named pipe which nobody was + reading. Thanks to Michael Biebl for reporting this bug. + Bugzilla entry: http://bugzilla.adiscon.com/show_bug.cgi?id=169 + [imported from 4.5.8] +--------------------------------------------------------------------------- +Version 5.3.6 [BETA] (rgerhards), 2010-01-13 +- bugfix: ompgsql did not properly check the server connection in + tryResume(), which could lead to rsyslog running in a thight loop +- bugfix: suspension during beginTransaction() was not properly handled + by rsyslog core +- bugfix: omfile output was only written when buffer was full, not at + end of transaction +- bugfix: commit transaction was not properly conveyed to message layer, + potentially resulting in non-message destruction and thus hangs +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +- bugfix: $CreateDirs variable not properly initialized, default thus + was random (but most often "on") [imported from v3] +- bugfix: ompgsql had problems with transaction support, what actually + rendered it unsuable. Thanks to forum user "horhe" for alerting me + on this bug and helping to debug/fix it! +- bugfix: memory leak when sending messages in zip-compressed format + Thanks to Naoya Nakazawa for analyzing this issue and providing a patch. +- worked around an issue where omfile failed to compile on 32 bit platforms + under some circumstances (this smells like a gcc problem, but a simple + solution was available). Thanks to Kenneth Marshall for some advice. + [backported from 5.5.x branch] +--------------------------------------------------------------------------- +Version 5.3.5 [BETA] (rgerhards), 2009-11-13 +- some light performance enhancement by replacing time() call with much + faster (at least under linux) gettimeofday() calls. +- some improvement of omfile performance with dynafiles + saved costly time() calls by employing a logical clock, which is + sufficient for the use case +- bugfix: omudpspoof miscalculated source and destination ports + while this was probably not noticed for source ports, it resulted in + almost all destination ports being wrong, except for the default port + of 514, which by virtue of its binary representation was calculated + correct (and probably thus the bug not earlier detected). +- bugfixes imported from earlier releases + * bugfix: named pipes did no longer work (they always got an open error) + this was a regression from the omfile rewrite in 4.5.0 + * bugfix(testbench): sequence check was not always performed correctly, + that could result in tests reporting success when they actually failed +- improved testbench: added tests for UDP forwarding and omudpspoof +- doc bugfix: omudpspoof had wrong config command names ("om" missing) +- bugfix [imported from 4.4.3]: $ActionExecOnlyOnceEveryInterval did + not work. +- [inport v4] improved testbench, contains now tcp and gzip test cases +- [import v4] added a so-called "On Demand Debug" mode, in which debug + output can be generated only after the process has started, but not right + from the beginning. This is assumed to be useful for hard-to-find bugs. + Also improved the doc on the debug system. +- bugfix: segfault on startup when -q or -Q option was given + [imported from v3-stable] +--------------------------------------------------------------------------- +Version 5.3.4 [DEVEL] (rgerhards), 2009-11-04 +- added the ability to create custom message parsers +- added $RulesetParser config directive that permits to bind specific + parsers to specific rulesets +- added omruleset output module, which provides great flexibility in + action processing. THIS IS A VERY IMPORTANT ADDITION, see its doc + for why. +- added the capability to have ruleset-specific main message queues + This offers considerable additional flexibility AND superior performance + (in cases where multiple inputs now can avoid lock contention) +- bugfix: correct default for escape ('#') character restored + This was accidently changed to '\\', thanks to David Lang for reporting +- bugfix(testbench): testcase did not properly wait for rsyslogd shutdown + thus some unpredictable behavior and a false negative test result + could occur. +--------------------------------------------------------------------------- +Version 5.3.3 [DEVEL] (rgerhards), 2009-10-27 +- simplified and thus speeded up the queue engine, also fixed some + potential race conditions (in very unusual shutdown conditions) + along the way. The threading model has seriously changes, so there may + be some regressions. +- enhanced test environment (inlcuding testbench): support for enhancing + probability of memory addressing failure by using non-NULL default + value for malloced memory (optional, only if requested by configure + option). This helps to track down some otherwise undetected issues + within the testbench. +- bugfix: potential abort if inputname property was not set + primarily a problem of imdiag +- bugfix: message processing states were not set correctly in all cases + however, this had no negative effect, as the message processing state + was not evaluated when a batch was deleted, and that was the only case + where the state could be wrong. +--------------------------------------------------------------------------- +Version 5.3.2 [DEVEL] (rgerhards), 2009-10-21 +- enhanced omfile to support transactional interface. This will increase + performance in many cases. +- added multi-ruleset support to imudp +- re-enabled input thread termination handling that does avoid thread + cancellation where possible. This provides a more reliable mode of + rsyslogd termination (canceling threads my result in not properly + freed resouces and potential later hangs, even though we perform + proper cancel handling in our code). This is part of an effort to + reduce thread cancellation as much as possible in rsyslog. + NOTE: the code previously written code for this functionality had a + subtle race condition. The new code solves that. +- enhanced immark to support non-cancel input module termination +- improved imudp so that epoll can be used in more environments, + fixed potential compile time problem if EPOLL_CLOEXEC is not available. +- some cleanup/slight improvement: + * changed imuxsock to no longer use deprecated submitAndParseMsg() IF + * changed submitAndParseMsg() interface to be a wrapper around the new + way of message creation/submission. This enables older plugins to be + used together with the new interface. The removal also enables us to + drop a lot of duplicate code, reducing complexity and increasing + maintainability. +- bugfix: segfault when starting up with an invalid .qi file for a disk queue + Failed for both pure disk as well as DA queues. Now, we emit an error + message and disable disk queueing facility. +- bugfix: potential segfault on messages with empty MSG part. This was a + recently introduced regression. +- bugfix: debug string larger than 1K were improperly displayed. Max size + is now 32K, and if a string is even longer it is meaningfully truncated. +--------------------------------------------------------------------------- +Version 5.3.1 [DEVEL] (rgerhards), 2009-10-05 +- added $AbortOnUncleanConfig directive - permits to prevent startup when + there are problems with the configuration file. See it's doc for + details. +- included some important fixes from v4-stable: + * bugfix: invalid handling of zero-sized messages + * bugfix: zero-sized UDP messages are no longer processed + * bugfix: random data could be appended to message + * bugfix: reverse lookup reduction logic in imudp do DNS queries too often +- bugfixes imported from 4.5.4: + * bugfix: potential segfault in stream writer on destruction + * bugfix: potential race in object loader (obj.c) during use/release + * bugfixes: potential problems in out file zip writer +--------------------------------------------------------------------------- +Version 5.3.0 [DEVEL] (rgerhards), 2009-09-14 +- begun to add simple GUI programs to gain insight into running rsyslogd + instances and help setup and troubleshooting (active via the + --enable-gui ./configure switch) +- changed imudp to utilize epoll(), where available. This shall provide + slightly better performance (just slightly because we called select() + rather infrequently on a busy system) +--------------------------------------------------------------------------- +Version 5.2.2 [v5-stable] (rgerhards), 2009-11-?? +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +--------------------------------------------------------------------------- +Version 5.2.1 [v5-stable] (rgerhards), 2009-11-02 +- bugfix [imported from 4.4.3]: $ActionExecOnlyOnceEveryInterval did + not work. +- bugfix: segfault on startup when -q or -Q option was given + [imported from v3-stable] +--------------------------------------------------------------------------- +Version 5.2.0 [v5-stable] (rgerhards), 2009-11-02 +This is a re-release of version 5.1.6 as stable after we did not get any bug +reports during the whole beta phase. Still, this first v5-stable may not be +as stable as one hopes for, I am not sure if we did not get bug reports +just because nobody tried it. Anyhow, we need to go forward and so we +have the initial v5-stable. +--------------------------------------------------------------------------- +Version 5.1.6 [v5-beta] (rgerhards), 2009-10-15 +- feature imports from v4.5.6 +- bugfix: potential race condition when queue worker threads were + terminated +- bugfix: solved potential (temporary) stall of messages when the queue was + almost empty and few new data added (caused testbench to sometimes hang!) +- fixed some race condition in testbench +- added more elaborate diagnostics to parts of the testbench +- bugfixes imported from 4.5.4: + * bugfix: potential segfault in stream writer on destruction + * bugfix: potential race in object loader (obj.c) during use/release + * bugfixes: potential problems in out file zip writer +- included some important fixes from 4.4.2: + * bugfix: invalid handling of zero-sized messages + * bugfix: zero-sized UDP messages are no longer processed + * bugfix: random data could be appended to message + * bugfix: reverse lookup reduction logic in imudp do DNS queries too often +--------------------------------------------------------------------------- +Version 5.1.5 [v5-beta] (rgerhards), 2009-09-11 +- added new config option $ActionWriteAllMarkMessages + this option permites to process mark messages under all circumstances, + even if an action was recently called. This can be useful to use mark + messages as a kind of heartbeat. +- added new config option $InputUnixListenSocketCreatePath + to permit the auto-creation of pathes to additional log sockets. This + turns out to be useful if they reside on temporary file systems and + rsyslogd starts up before the daemons that create these sockets + (rsyslogd always creates the socket itself if it does not exist). +- added $LogRSyslogStatusMessages configuration directive + permitting to turn off rsyslog start/stop/HUP messages. See Debian + ticket http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=463793 +- bugfix: hostnames with dashes in them were incorrectly treated as + malformed, thus causing them to be treated as TAG (this was a regression + introduced from the "rfc3164 strict" change in 4.5.0). Testbench has been + updated to include a smaple message with a hostname containing a dash. +- bugfix: strings improperly reused, resulting in some message properties + be populated with strings from previous messages. This was caused by + an improper predicate check. +- added new config directive $omfileForceChown [import from 4.7.0] +--------------------------------------------------------------------------- +Version 5.1.4 [DEVEL] (rgerhards), 2009-08-20 +- legacy syslog parser changed so that it now accepts date stamps in + wrong case. Some devices seem to create them and I do not see any harm + in supporting that. +- added $InputTCPMaxListeners directive - permits to specify how many + TCP servers shall be possible (default is 20). +- bugfix: memory leak with some input modules. Those inputs that + use parseAndSubmitMsg() leak two small memory blocks with every message. + Typically, those process only relatively few messages, so the issue + does most probably not have any effect in practice. +- bugfix: if tcp listen port could not be created, no error message was + emitted +- bugfix: discard action did not work (did not discard messages) +- bugfix: discard action caused segfault +- bugfix: potential segfault in output file writer (omfile) + In async write mode, we use modular arithmetic to index the output + buffer array. However, the counter variables accidently were signed, + thus resulting in negative indizes after integer overflow. That in turn + could lead to segfaults, but was depending on the memory layout of + the instance in question (which in turn depended on a number of + variables, like compile settings but also configuration). The counters + are now unsigned (as they always should have been) and so the dangling + mis-indexing does no longer happen. This bug potentially affected all + installations, even if only some may actually have seen a segfault. +--------------------------------------------------------------------------- +Version 5.1.3 [DEVEL] (rgerhards), 2009-07-28 +- architecture change: queue now always has at least one worker thread + if not running in direct mode. Previous versions could run without + any active workers. This simplifies the code at a very small expense. + See v5 compatibility note document for more in-depth discussion. +- enhance: UDP spoofing supported via new output module omudpspoof + See the omudpspoof documentation for details and samples +- bugfix: message could be truncated after TAG, often when forwarding + This was a result of an internal processing error if maximum field + sizes had been specified in the property replacer. +- bugfix: minor static memory leak while reading configuration + did NOT leak based on message volume +- internal: added ability to terminate input modules not via pthread_cancel + but an alternate approach via pthread_kill. This is somewhat safer as we + do not need to think about the cancel-safeness of all libraries we use. + However, not all inputs can easily supported, so this now is a feature + that can be requested by the input module (the most important ones + request it). +--------------------------------------------------------------------------- +Version 5.1.2 [DEVEL] (rgerhards), 2009-07-08 +- bugfix: properties inputname, fromhost, fromhost-ip, msg were lost when + working with disk queues +- some performance enhancements +- bugfix: abort condition when RecvFrom was not set and message reduction + was on. Happend e.g. with imuxsock. +- added $klogConsoleLogLevel directive which permits to set a new + console log level while rsyslog is active +- some internal code cleanup +--------------------------------------------------------------------------- +Version 5.1.1 [DEVEL] (rgerhards), 2009-07-03 +- bugfix: huge memory leak in queue engine (made rsyslogd unusable in + production). Occured if at least one queue was in direct mode + (the default for action queues) +- imported many performance optimizations from v4-devel (4.5.0) +- bugfix: subtle (and usually irrelevant) issue in timout processing + timeout could be one second too early if nanoseconds wrapped +- set a more sensible timeout for shutdow, now 1.5 seconds to complete + processing (this also removes those cases where the shutdown message + was not written because the termination happened before it) +--------------------------------------------------------------------------- +Version 5.1.0 [DEVEL] (rgerhards), 2009-05-29 + +*********************************** NOTE ********************************** +The v5 versions of rsyslog feature a greatly redesigned queue engine. The +major theme for the v5 release is twofold: + +a) greatly improved performance +b) enable audit-grade processing + +Here, audit-grade processing means that rsyslog, if used together with +audit-grade transports and configured correctly, will never lose messages +that already have been acknowledged, not even in fatal failure cases like +sudden loss of power. + +Note that large parts of rsyslog's important core components have been +restructured to support these design goals. As such, early versions of +the engine will probably be less stable than the v3/v4 engine. + +Also note that the initial versions do not cover all and everything. As +usual, the code will evolve toward the final goal as version numbers +increase. +*********************************** NOTE ********************************** + +- redesigned queue engine so that it supports ultra-reliable operations + This resulted in a rewrite of large parts. The new capability can be + used to build audit-grade systems on the basis of rsyslog. +- added $MainMsgQueueDequeueBatchSize and $ActionQueueDequeueBatchSize + configuration directives +- implemented a new transactional output module interface which provides + superior performance (for databases potentially far superior performance) +- increased ompgsql performance by adapting to new transactional + output module interface +--------------------------------------------------------------------------- +Version 4.7.0 [v4-devel] (rgerhards), 2009-09-?? +- added function getenv() to RainerScript +- added new config option $InputUnixListenSocketCreatePath + to permit the auto-creation of pathes to additional log sockets. This + turns out to be useful if they reside on temporary file systems and + rsyslogd starts up before the daemons that create these sockets + (rsyslogd always creates the socket itself if it does not exist). +- added $LogRSyslogStatusMessages configuration directive + permitting to turn off rsyslog start/stop/HUP messages. See Debian + ticket http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=463793 +- added new config directive $omfileForceChown to (try to) fix some broken + system configs. + See ticket for details: http://bugzilla.adiscon.com/show_bug.cgi?id=150 +- added $EscapeControlCharacterTab config directive + Thanks to Jonathan Bond-Caron for the patch. +- added option to use unlimited-size select() calls + Thanks to varmjofekoj for the patch +- debugondemand mode caused backgrounding to fail - close to a bug, but I'd + consider the ability to background in this mode a new feature... +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +- imported changes from 4.5.7 and below +- bugfix: potential segfault when -p command line option was used + Thanks for varmojfekoj for pointing me at this bug. +- imported changes from 4.5.6 and below +--------------------------------------------------------------------------- +Version 4.6.1 [v4-stable] (rgerhards), 2010-02-?? +- re-enabled old pipe output (using new module ompipe, built-in) after + some problems with pipes (and especially in regard to xconsole) were + discovered. Thanks to Michael Biebl for reporting the issues. +- bugfix: fixed problem that caused compilation on FreeBSD 9.0 to fail. + bugtracker: http://bugzilla.adiscon.com/show_bug.cgi?id=181 + Thanks to Christiano for reporting. +- bugfix: comment char ('#') in literal terminated script parsing + and thus could not be used. + but tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=119 + [merged in from v3.22.2] +--------------------------------------------------------------------------- +Version 4.6.0 [v4-stable] (rgerhards), 2010-02-24 +*************************************************************************** +* This is a new stable v4 version. It contains all fixes and enhancements * +* made during the 4.5.x phase as well as those listed below. * +* Note: this version is scheduled to conclude the v4 development process. * +* Do not expect any more new developments in v4. The focus is now * +* on v5 (what also means we have a single devel branch again). * +* ("development" means new feature development, bug fixes are of * +* course provided for v4-stable) * +*************************************************************************** +- improved testbench to contain samples for totally malformed messages + which miss parts of the message content +- bugfix: some malformed messages could lead to a missing LF inside files + or some other missing parts of the template content. +- bugfix: if a message ended immediately with a hostname, the hostname + was mistakenly interpreted as TAG, and localhost be used as hostname +- bugfix: message without MSG part could case a segfault + [backported from v5 commit 98d1ed504ec001728955a5bcd7916f64cd85f39f] + This actually was a "recent" regression, but I did not realize that it + was introduced by the performance optimization in v4-devel. Shame on + me for having two devel versions at the same time... +--------------------------------------------------------------------------- +Version 4.5.8 [v4-beta] (rgerhards), 2010-02-10 +- enhanced doc for using PostgreSQL + Thanks to Marc Schiffbauer for the new/updated doc +- bugfix: property replacer returned invalid parameters under some (unusual) + conditions. In extreme cases, this could lead to garbled logs and/or + a system failure. +- bugfix: invalid length returned (often) when using regular expressions + inside the property replacer +- bugfix: submatch regex in property replacer did not honor "return 0 on + no match" config case +- bugfix: imuxsock incorrectly stated inputname "imudp" + Thanks to Ryan Lynch for reporting this. +- (slightly) enhanced support for FreeBSD by setting _PATH_MODDIR to + the correct value on FreeBSD. + Thanks to Cristiano for the patch. +- bugfix: -d did not enable display of debug messages + regression from introduction of "debug on demand" mode + Thanks to Michael Biebl for reporting this bug +- bugfix: blanks inside file names did not terminate file name parsing. + This could reslult in the whole rest of a line (including comments) + to be treated as file name in "write to file" actions. + Thanks to Jack for reporting this issue. +- bugfix: rsyslog hang when writing to a named pipe which nobody was + reading. Thanks to Michael Biebl for reporting this bug. + Bugzilla entry: http://bugzilla.adiscon.com/show_bug.cgi?id=169 +- bugfix: potential segfaults during queue shutdown + (bugs require certain non-standard settings to appear) + Thanks to varmojfekoj for the patch +--------------------------------------------------------------------------- +Version 4.5.7 [v4-beta] (rgerhards), 2009-11-18 +- added a so-called "On Demand Debug" mode, in which debug output can + be generated only after the process has started, but not right from + the beginning. This is assumed to be useful for hard-to-find bugs. + Also improved the doc on the debug system. +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +- bugfix: hostname accidently set to IP address for some message sources, + for example imudp. Thanks to Anton for reporting this bug. +- bugfix [imported from 4.4.3]: $ActionExecOnlyOnceEveryInterval did + not work. +--------------------------------------------------------------------------- +Version 4.5.6 [v4-beta] (rgerhards), 2009-11-05 +- bugfix: named pipes did no longer work (they always got an open error) + this was a regression from the omfile rewrite in 4.5.0 +- bugfix(minor): diag function returned wrong queue memeber count + for the main queue if an active DA queue existed. This had no relevance + to real deployments (assuming they are not running the debug/diagnostic + module...), but sometimes caused grief and false alerts in the + testbench. +- included some important fixes from v4-stable: + * bugfix: invalid handling of zero-sized messages + * bugfix: zero-sized UDP messages are no longer processed + * bugfix: random data could be appended to message + * bugfix: reverse lookup reduction logic in imudp do DNS queries too often +- bugfix(testbench): testcase did not properly wait for rsyslod shutdown + thus some unpredictable behavior and a false negative test result + could occur. [BACKPORTED from v5] +- bugfix(testbench): sequence check was not always performed correctly, + that could result in tests reporting success when they actually failed +--------------------------------------------------------------------------- +Version 4.5.5 [v4-beta] (rgerhards), 2009-10-21 +- added $InputTCPServerNotifyOnConnectionClose config directive + see doc for details +- bugfix: debug string larger than 1K were improperly displayed. Max size + is now 32K +- bugfix: invalid storage class selected for some size config parameters. + This resulted in wrong values. The most prominent victim was the + directory creation mode, which was set to zero in some cases. For + details, see related blog post: + http://blog.gerhards.net/2009/10/another-note-on-hard-to-find-bugs.html +--------------------------------------------------------------------------- +Version 4.5.4 [v4-beta] (rgerhards), 2009-09-29 +- bugfix: potential segfault in stream writer on destruction + Most severely affected omfile. The problem was that some buffers were + freed before the asynchronous writer thread was shut down. So the + writer thread accessed invalid data, which may even already be + overwritten. Symptoms (with omfile) were segfaults, grabled data + and files with random names placed around the file system (most + prominently into the root directory). Special thanks to Aaron for + helping to track this down. +- bugfix: potential race in object loader (obj.c) during use/release + of object interface +- bugfixes: potential problems in out file zip writer. Problems could + lead to abort and/or memory leak. The module is now hardened in a very + conservative way, which is sub-optimal from a performance point of view. + This should be improved if it has proven reliable in practice. +--------------------------------------------------------------------------- +Version 4.5.3 [v4-beta] (rgerhards), 2009-09-17 +- bugfix: repeated messages were incorrectly processed + this could lead to loss of the repeated message content. As a side- + effect, it could probably also be possible that some segfault occurs + (quite unlikely). The root cause was that some counters introduced + during the malloc optimizations were not properly duplicated in + MsgDup(). Note that repeated message processing is not enabled + by default. +- bugfix: message sanitation had some issues: + - control character DEL was not properly escaped + - NUL and LF characters were not properly stripped if no control + character replacement was to be done + - NUL characters in the message body were silently dropped (this was + a regeression introduced by some of the recent optimizations) +- bugfix: strings improperly reused, resulting in some message properties + be populated with strings from previous messages. This was caused by + an improper predicate check. [backported from v5] +- fixed some minor portability issues +- bugfix: reverse lookup reduction logic in imudp do DNS queries too often + [imported from 4.4.2] +--------------------------------------------------------------------------- +Version 4.5.2 [v4-beta] (rgerhards), 2009-08-21 +- legacy syslog parser changed so that it now accepts date stamps in + wrong case. Some devices seem to create them and I do not see any harm + in supporting that. +- added $InputTCPMaxListeners directive - permits to specify how many + TCP servers shall be possible (default is 20). +- bugfix: memory leak with some input modules. Those inputs that + use parseAndSubmitMsg() leak two small memory blocks with every message. + Typically, those process only relatively few messages, so the issue + does most probably not have any effect in practice. +- bugfix: if tcp listen port could not be created, no error message was + emitted +- bugfix: potential segfault in output file writer (omfile) + In async write mode, we use modular arithmetic to index the output + buffer array. However, the counter variables accidently were signed, + thus resulting in negative indizes after integer overflow. That in turn + could lead to segfaults, but was depending on the memory layout of + the instance in question (which in turn depended on a number of + variables, like compile settings but also configuration). The counters + are now unsigned (as they always should have been) and so the dangling + mis-indexing does no longer happen. This bug potentially affected all + installations, even if only some may actually have seen a segfault. +- bugfix: hostnames with dashes in them were incorrectly treated as + malformed, thus causing them to be treated as TAG (this was a regression + introduced from the "rfc3164 strict" change in 4.5.0). +--------------------------------------------------------------------------- +Version 4.5.1 [DEVEL] (rgerhards), 2009-07-15 +- CONFIG CHANGE: $HUPisRestart default is now "off". We are doing this + to support removal of restart-type HUP in v5. +- bugfix: fromhost-ip was sometimes truncated +- bugfix: potential segfault when zip-compressed syslog records were + received (double free) +- bugfix: properties inputname, fromhost, fromhost-ip, msg were lost when + working with disk queues +- performance enhancement: much faster, up to twice as fast (depending + on configuration) +- bugfix: abort condition when RecvFrom was not set and message reduction + was on. Happend e.g. with imuxsock. +- added $klogConsoleLogLevel directive which permits to set a new + console log level while rsyslog is active +- bugfix: message could be truncated after TAG, often when forwarding + This was a result of an internal processing error if maximum field + sizes had been specified in the property replacer. +- added ability for the TCP output action to "rebind" its send socket after + sending n messages (actually, it re-opens the connection, the name is + used because this is a concept very similiar to $ActionUDPRebindInterval). + New config directive $ActionSendTCPRebindInterval added for the purpose. + By default, rebinding is disabled. This is considered useful for load + balancers. +- testbench improvements +--------------------------------------------------------------------------- +Version 4.5.0 [DEVEL] (rgerhards), 2009-07-02 +- activation order of inputs changed, they are now activated only after + privileges are dropped. Thanks to Michael Terry for the patch. +- greatly improved performance +- greatly reduced memory requirements of msg object + to around half of the previous demand. This means that more messages can + be stored in core! Due to fewer cache misses, this also means some + performance improvement. +- improved config error messages: now contain a copy of the config line + that (most likely) caused the error +- reduced max value for $DynaFileCacheSize to 1,000 (the former maximum + of 10,000 really made no sense, even 1,000 is very high, but we like + to keep the user in control ;)). +- added capability to fsync() queue disk files for enhanced reliability + (also add's speed, because you do no longer need to run the whole file + system in sync mode) +- more strict parsing of the hostname in rfc3164 mode, hopefully + removes false positives (but may cause some trouble with hostname + parsing). For details, see this bug tracker: + http://bugzilla.adiscon.com/show_bug.cgi?id=126 +- omfile rewrite to natively support zip files (includes large extension + of the stream class) +- added configuration commands (see doc for explanations) + * $OMFileZipLevel + * $OMFileIOBufferSize + * $OMFileFlushOnTXEnd + * $MainMsgQueueSyncQueueFiles + * $ActionQueueSyncQueueFiles +- done some memory accesses explicitely atomic +- bugfix: subtle (and usually irrelevant) issue in timout processing + timeout could be one second too early if nanoseconds wrapped +- set a more sensible timeout for shutdow, now 1.5 seconds to complete + processing (this also removes those cases where the shutdown message + was not written because the termination happened before it) +- internal bugfix: object pointer was only reset to NULL when an object + was actually destructed. This most likely had no effect to existing code, + but it may also have caused trouble in remote cases. Similarly, the fix + may also cause trouble... +- bugfix: missing initialization during timestamp creation + This could lead to timestamps written in the wrong format, but not to + an abort +--------------------------------------------------------------------------- +Version 4.4.3 [v4-stable] (rgerhards), 2009-10-?? +- bugfix: several smaller bugs resolved after flexelint review + Thanks to varmojfekoj for the patch. +- bugfix: $ActionExecOnlyOnceEveryInterval did not work. + This was a regression from the time() optimizations done in v4. + Bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=143 + Thanks to Klaus Tachtler for reporting this bug. +- bugfix: potential segfault on queue shutdown + Thanks to varmojfekoj for the patch. +- bugfix: potential hang condition on queue shutdown + [imported from v3-stable] +- bugfix: segfault on startup when -q or -Q option was given + [imported from v3-stable] +--------------------------------------------------------------------------- +Version 4.4.2 [v4-stable] (rgerhards), 2009-10-09 +- bugfix: invalid handling of zero-sized messages, could lead to mis- + addressing and potential memory corruption/segfault +- bugfix: zero-sized UDP messages are no longer processed + until now, they were forwarded to processing, but this makes no sense + Also, it looks like the system seems to provide a zero return code + on a UDP recvfrom() from time to time for some internal reasons. These + "receives" are now silently ignored. +- bugfix: random data could be appended to message, possibly causing + segfaults +- bugfix: reverse lookup reduction logic in imudp do DNS queries too often + A comparison was done between the current and the former source address. + However, this was done on the full sockaddr_storage structure and not + on the host address only. This has now been changed for IPv4 and IPv6. + The end result of this bug could be a higher UDP message loss rate than + necessary (note that UDP message loss can not totally be avoided due + to the UDP spec) +--------------------------------------------------------------------------- +Version 4.4.1 [v4-stable] (rgerhards), 2009-09-02 +- features requiring Java are automatically disabled if Java is not + present (thanks to Michael Biebl for his help!) +- bugfix: invalid double-quoted PRI, among others in outgoing messages + This causes grief with all receivers. + Bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=147 +- bugfix: Java testing tools were required, even if testbench was disabled + This resulted in build errors if no Java was present on the build system, + even though none of the selected option actually required Java. + (I forgot to backport a similar fix to newer releases). +- bugfix (backport): omfwd segfault + Note that the orginal (higher version) patch states this happens only + when debugging mode is turned on. That statement is wrong: if debug + mode is turned off, the message is not being emitted, but the division + by zero in the actual parameters still happens. +--------------------------------------------------------------------------- +Version 4.4.0 [v4-stable] (rgerhards), 2009-08-21 +- bugfix: stderr/stdout were not closed to be able to emit error messages, + but this caused ssh sessions to hang. Now we close them after the + initial initialization. See forum thread: + http://kb.monitorware.com/controlling-terminal-issues-t9875.html +- bugfix: sending syslog messages with zip compression did not work +--------------------------------------------------------------------------- +Version 4.3.2 [v4-beta] (rgerhards), 2009-06-24 +- removed long-obsoleted property UxTradMsg +- added a generic network stream server (in addition to rather specific + syslog tcp server) +- added ability for the UDP output action to rebind its send socket after + sending n messages. New config directive $ActionSendUDPRebindInterval + added for the purpose. By default, rebinding is disabled. This is + considered useful for load balancers. - bugfix: imdiag/imtcp had a race condition +- improved testbench (now much better code design and reuse) +- added config switch --enable-testbench=no to turn off testbench --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) @@ -45,9 +753,10 @@ Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 execution. The most probable case for such a case would be invalid sql inside the template, and this is now much easier to diagnose. --------------------------------------------------------------------------- -Version 4.1.8 [BETA] (rgerhards), 2009-04-?? +Version 4.2.0 [v4-stable] (rgerhards), 2009-06-23 - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- imported all patches from 3.22.1 as of today (see below) - bugfix: compile problems in im3195 --------------------------------------------------------------------------- Version 4.1.7 [BETA] (rgerhards), 2009-04-22 @@ -198,9 +907,57 @@ version before switching to this one. - bugfix: memory leak in ompgsql Thanks to Ken for providing the patch --------------------------------------------------------------------------- -Version 3.22.1 [v3-stable] (rgerhards), 2009-04-?? +Version 3.22.2 [v3-stable] (rgerhards), 2009-07-?? +- bugfix: comment char ('#') in literal terminated script parsing + and thus could not be used. + but tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=119 +- enhance: imrelp now also provides remote peer's IP address + [if librelp != 1.0.0 is used] +- bugfix: sending syslog messages with zip compression did not work +- bugfix: potential hang condition on queue shutdown +- bugfix: segfault on startup when -q or -Q option was given + bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=157 + Thanks to Jonas Nogueira for reporting this bug. +- clarified use of $ActionsSendStreamDriver[AuthMode/PermittedPeers] + in doc set (require TLS drivers) +- bugfix: $CreateDirs variable not properly initialized, default thus + was random (but most often "on") +- bugfix: potential segfault when -p command line option was used + thanks to varmojfekoj for pointing me at this bug +--------------------------------------------------------------------------- +Version 3.22.1 [v3-stable] (rgerhards), 2009-07-02 +- bugfix: invalid error message issued if $inlcudeConfig was on an empty + set of files (e.g. *.conf, where none such files existed) + thanks to Michael Biebl for reporting this bug +- bugfix: when run in foreground (but not in debug mode), a + debug message ("DoDie called") was emitted at shutdown. Removed. + thanks to Michael Biebl for reporting this bug +- bugfix: some garbagge was emitted to stderr on shutdown. This + garbage consisted of file names, which were written during + startup (key point: not a pointer error) + thanks to Michael Biebl for reporting this bug +- bugfix: startup and shutdown message were emitted to stdout + thanks to Michael Biebl for reporting this bug +- bugfix: error messages were not emitted to stderr in forked mode + (stderr and stdo are now kept open across forks) +- bugfix: internal messages were emitted to whatever file had fd2 when + rsyslogd ran in forked mode (as usual!) + Thanks to varmojfekoj for the patch +- small enhancement: config validation run now exits with code 1 if an + error is detected. This change is considered important but small enough + to apply it directly to the stable version. [But it is a border case, + the change requires more code than I had hoped. Thus I have NOT tried + to actually catch all cases, this is left for the current devel + releases, if necessary] - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- bugfix: potential segfault issue when multiple $UDPServerRun directives + are specified. Thanks to Michael Biebl for helping to debug this one. +- relaxed GnuTLS version requirement to 1.4.0 after confirmation from the + field that this version is sufficient +- bugfix: parser did not properly handle empty structured data +- bugfix: invalid mutex release in msg.c (detected under thread debugger, + seems not to have any impact on actual deployments) --------------------------------------------------------------------------- Version 3.22.0 [v3-stable] (rgerhards), 2009-04-21 This is the first stable release that includes the full functionality @@ -1315,6 +2072,7 @@ Version 2.0.8 V2-STABLE (rgerhards), 2008-??-?? connection broke, but not if there was a problem with statement execution. The most probable case for such a case would be invalid sql inside the template, and this is now much easier to diagnose. +- doc bugfix: default for $DirCreateMode incorrectly stated --------------------------------------------------------------------------- Version 2.0.7 V2-STABLE (rgerhards), 2008-04-14 - bugfix: the default for $DirCreateMode was 0644, and as such wrong. diff --git a/Makefile.am b/Makefile.am index e991f323..52a716fd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,6 @@ sbin_PROGRAMS = pkglib_LTLIBRARIES = - if ENABLE_INET pkglib_LTLIBRARIES += lmtcpsrv.la lmtcpclt.la # @@ -84,6 +83,10 @@ if ENABLE_SNMP SUBDIRS += plugins/omsnmp endif +if ENABLE_CUST1 +SUBDIRS += plugins/cust1 +endif + if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif @@ -92,6 +95,18 @@ if ENABLE_OMSTDOUT SUBDIRS += plugins/omstdout endif +if ENABLE_OMRULESET +SUBDIRS += plugins/omruleset +endif + +if ENABLE_OMDBALERTING +SUBDIRS += plugins/omdbalerting +endif + +if ENABLE_OMUDPSPOOF +SUBDIRS += plugins/omudpspoof +endif + if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif @@ -120,6 +135,10 @@ if ENABLE_ORACLE SUBDIRS += plugins/omoracle endif +if ENABLE_GUI +SUBDIRS += java +endif + # tests are added as last element, because tests may need different # modules that need to be generated first SUBDIRS += tests @@ -131,5 +150,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omruleset --enable-omprog --enable-imdiag --enable-shave --enable-memcheck ACLOCAL_AMFLAGS = -I m4 @@ -4,7 +4,7 @@ * * File begun on 2007-08-06 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -42,10 +42,15 @@ #include "cfsysline.h" #include "srUtils.h" #include "errmsg.h" +#include "batch.h" +#include "wti.h" #include "datetime.h" +#include "unicode-helper.h" + +#define NO_TIME_PROVIDED 0 /* indicate we do not provide any cached time */ /* forward definitions */ -rsRetVal actionCallDoAction(action_t *pAction, msg_t *pMsg); +static rsRetVal processBatchMain(action_t *pAction, batch_t *pBatch, int*); /* object static data (once for all instances) */ /* TODO: make this an object! DEFobjStaticHelpers -- rgerhards, 2008-03-05 */ @@ -54,16 +59,19 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +static int iActExecOnceInterval = 0; /* execute action once every nn seconds */ static int iActExecEveryNthOccur = 0; /* execute action every n-th occurence (0,1=always) */ static time_t iActExecEveryNthOccurTO = 0; /* timeout for n-occurence setting (in seconds, 0=never) */ static int glbliActionResumeInterval = 30; int glbliActionResumeRetryCount = 0; /* how often should suspended actions be retried? */ static int bActionRepMsgHasMsg = 0; /* last messsage repeated... has msg fragment in it */ +static int bActionWriteAllMarkMsgs = FALSE; /* should all mark messages be unconditionally written? */ static uchar *pszActionName; /* short name for the action */ -/* main message queue and its configuration parameters */ +/* action queue and its configuration parameters */ static queueType_t ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ static int iActionQueueSize = 1000; /* size of the main message queue above */ +static int iActionQueueDeqBatchSize = 16; /* batch size for action queues */ static int iActionQHighWtrMark = 800; /* high water mark for disk-assisted queues */ static int iActionQLowWtrMark = 200; /* low water mark for disk-assisted queues */ static int iActionQDiscardMark = 9800; /* begin to discard messages */ @@ -72,6 +80,7 @@ static int iActionQueueNumWorkers = 1; /* number of worker threads for the mm static uchar *pszActionQFName = NULL; /* prefix for the main message queue file */ static int64 iActionQueMaxFileSize = 1024*1024; static int iActionQPersistUpdCnt = 0; /* persist queue info every n updates */ +static int bActionQSyncQeueFiles = 0; /* sync queue files */ static int iActionQtoQShutdown = 0; /* queue shutdown */ static int iActionQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iActionQtoEnq = 2000; /* timeout for queue enque */ @@ -120,7 +129,7 @@ getActNow(action_t *pThis) { assert(pThis != NULL); if(pThis->tActNow == -1) { - pThis->tActNow = time(NULL); /* good time call - the only one done */ + pThis->tActNow = datetime.GetTime(NULL); /* good time call - the only one done */ if(pThis->tLastExec > pThis->tActNow) { /* if we are traveling back in time, reset tLastExec */ pThis->tLastExec = (time_t) 0; @@ -144,6 +153,7 @@ actionResetQueueParams(void) ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ iActionQueueSize = 1000; /* size of the main message queue above */ + iActionQueueDeqBatchSize = 16; /* default batch size */ iActionQHighWtrMark = 800; /* high water mark for disk-assisted queues */ iActionQLowWtrMark = 200; /* low water mark for disk-assisted queues */ iActionQDiscardMark = 9800; /* begin to discard messages */ @@ -151,6 +161,7 @@ actionResetQueueParams(void) iActionQueueNumWorkers = 1; /* number of worker threads for the mm queue above */ iActionQueMaxFileSize = 1024*1024; iActionQPersistUpdCnt = 0; /* persist queue info every n updates */ + bActionQSyncQeueFiles = 0; iActionQtoQShutdown = 0; /* queue shutdown */ iActionQtoActShutdown = 1000; /* action shutdown (in phase 2) */ iActionQtoEnq = 2000; /* timeout for queue enque */ @@ -176,6 +187,7 @@ actionResetQueueParams(void) */ rsRetVal actionDestruct(action_t *pThis) { + int i; DEFiRet; ASSERT(pThis != NULL); @@ -193,6 +205,36 @@ rsRetVal actionDestruct(action_t *pThis) pthread_mutex_destroy(&pThis->mutActExec); d_free(pThis->pszName); d_free(pThis->ppTpl); + + /* message ptr cleanup */ + for(i = 0 ; i < pThis->iNumTpls ; ++i) { + if(pThis->ppMsgs[i] != NULL) { + switch(pThis->eParamPassing) { + case ACT_ARRAY_PASSING: +#if 0 /* later, as an optimization. So far, we do the cleanup after each message */ + iArr = 0; + while(((char **)pThis->ppMsgs[i])[iArr] != NULL) { + d_free(((char **)pThis->ppMsgs[i])[iArr++]); + ((char **)pThis->ppMsgs[i])[iArr++] = NULL; + } + d_free(pThis->ppMsgs[i]); + pThis->ppMsgs[i] = NULL; +#endif + break; + case ACT_STRING_PASSING: + d_free(pThis->ppMsgs[i]); + break; + case ACT_MSG_PASSING: + /* No cleanup needed in this case */ + break; + default: + assert(0); + } + } + } + d_free(pThis->ppMsgs); + d_free(pThis->lenMsgs); + d_free(pThis); RETiRet; @@ -209,13 +251,10 @@ rsRetVal actionConstruct(action_t **ppThis) ASSERT(ppThis != NULL); - if((pThis = (action_t*) calloc(1, sizeof(action_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - + CHKmalloc(pThis = (action_t*) calloc(1, sizeof(action_t))); pThis->iResumeInterval = glbliActionResumeInterval; pThis->iResumeRetryCount = glbliActionResumeRetryCount; - pThis->tLastOccur = time(NULL); /* done once per action on startup only */ + pThis->tLastOccur = datetime.GetTime(NULL); /* done once per action on startup only */ pthread_mutex_init(&pThis->mutActExec, NULL); SYNC_OBJ_TOOL_INIT(pThis); @@ -255,7 +294,8 @@ actionConstructFinalize(action_t *pThis) * to be run on multiple threads. So far, this is forbidden by the interface * spec. -- rgerhards, 2008-01-30 */ - CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, (rsRetVal (*)(void*,void*))actionCallDoAction)); + CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, + (rsRetVal (*)(void*, batch_t*, int*))processBatchMain)); obj.SetName((obj_t*) pThis->pQueue, pszQName); /* ... set some properties ... */ @@ -270,9 +310,11 @@ actionConstructFinalize(action_t *pThis) qqueueSetpUsr(pThis->pQueue, pThis); setQPROP(qqueueSetsizeOnDiskMax, "$ActionQueueMaxDiskSpace", iActionQueMaxDiskSpace); + setQPROP(qqueueSetiDeqBatchSize, "$ActionQueueDequeueBatchSize", iActionQueueDeqBatchSize); setQPROP(qqueueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); setQPROPstr(qqueueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); setQPROP(qqueueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); + setQPROP(qqueueSetbSyncQueueFiles, "$ActionQueueSyncQueueFiles", bActionQSyncQeueFiles); setQPROP(qqueueSettoQShutdown, "$ActionQueueTimeoutShutdown", iActionQtoQShutdown ); setQPROP(qqueueSettoActShutdown, "$ActionQueueTimeoutActionCompletion", iActionQtoActShutdown); setQPROP(qqueueSettoWrkShutdown, "$ActionQueueWorkerTimeoutThreadShutdown", iActionQtoWrkShutdown); @@ -295,7 +337,7 @@ actionConstructFinalize(action_t *pThis) CHKiRet(qqueueStart(pThis->pQueue)); - dbgprintf("Action %p: queue %p created\n", pThis, pThis->pQueue); + DBGPRINTF("Action %p: queue %p created\n", pThis, pThis->pQueue); /* and now reset the queue params (see comment in its function header!) */ actionResetQueueParams(); @@ -305,87 +347,250 @@ finalize_it: } -/* set an action back to active state -- rgerhards, 2007-08-02 + +/* set the global resume interval + */ +rsRetVal actionSetGlobalResumeInterval(int iNewVal) +{ + glbliActionResumeInterval = iNewVal; + return RS_RET_OK; +} + + +/* returns the action state name in human-readable form + * returned string must not be modified. + * rgerhards, 2009-05-07 + */ +static uchar *getActStateName(action_t *pThis) +{ + switch(pThis->eState) { + case ACT_STATE_RDY: + return (uchar*) "rdy"; + case ACT_STATE_ITX: + return (uchar*) "itx"; + case ACT_STATE_RTRY: + return (uchar*) "rtry"; + case ACT_STATE_SUSP: + return (uchar*) "susp"; + case ACT_STATE_DIED: + return (uchar*) "died"; + case ACT_STATE_COMM: + return (uchar*) "comm"; + default: + return (uchar*) "ERROR/UNKNWON"; + } +} + + +/* returns a suitable return code based on action state + * rgerhards, 2009-05-07 */ -static rsRetVal actionResume(action_t *pThis) +static rsRetVal getReturnCode(action_t *pThis) { DEFiRet; ASSERT(pThis != NULL); - pThis->bSuspended = 0; + switch(pThis->eState) { + case ACT_STATE_RDY: + iRet = RS_RET_OK; + break; + case ACT_STATE_ITX: + if(pThis->bHadAutoCommit) { + pThis->bHadAutoCommit = 0; /* auto-reset */ + iRet = RS_RET_PREVIOUS_COMMITTED; + } else { + iRet = RS_RET_DEFER_COMMIT; + } + break; + case ACT_STATE_RTRY: + iRet = RS_RET_SUSPENDED; + break; + case ACT_STATE_SUSP: + case ACT_STATE_DIED: + iRet = RS_RET_ACTION_FAILED; + break; + default: + DBGPRINTF("Invalid action engine state %d, program error\n", + (int) pThis->eState); + iRet = RS_RET_ERR; + break; + } RETiRet; } -/* set the global resume interval +/* set the action to a new state + * rgerhards, 2007-08-02 */ -rsRetVal actionSetGlobalResumeInterval(int iNewVal) +static inline void actionSetState(action_t *pThis, action_state_t newState) { - glbliActionResumeInterval = iNewVal; - return RS_RET_OK; + pThis->eState = newState; + DBGPRINTF("Action %p transitioned to state: %s\n", pThis, getActStateName(pThis)); +} + +/* Handles the transient commit state. So far, this is + * mostly a dummy... + * rgerhards, 2007-08-02 + */ +static void actionCommitted(action_t *pThis) +{ + actionSetState(pThis, ACT_STATE_RDY); +} + + +/* set action to "rtry" state. + * rgerhards, 2007-08-02 + */ +static void actionRetry(action_t *pThis) +{ + actionSetState(pThis, ACT_STATE_RTRY); +} + + +/* Disable action, this means it will never again be usable + * until rsyslog is reloaded. Use only as a last resort, but + * depends on output module. + * rgerhards, 2007-08-02 + */ +static void actionDisable(action_t *pThis) +{ + actionSetState(pThis, ACT_STATE_DIED); } -/* suspend an action -- rgerhards, 2007-08-02 +/* Suspend action, this involves changing the acton state as well + * as setting the next retry time. + * if we have more than 10 retries, we prolong the + * retry interval. If something is really stalled, it will + * get re-tried only very, very seldom - but that saves + * CPU time. TODO: maybe a config option for that? + * rgerhards, 2007-08-02 */ -static rsRetVal actionSuspend(action_t *pThis, time_t tNow) +static inline void actionSuspend(action_t *pThis, time_t ttNow) { + if(ttNow == NO_TIME_PROVIDED) + datetime.GetTime(&ttNow); + pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); + actionSetState(pThis, ACT_STATE_SUSP); + DBGPRINTF("earliest retry=%d\n", (int) pThis->ttResumeRtry); +} + + +/* actually do retry processing. Note that the function receives a timestamp so + * that we do not need to call the (expensive) time() API. + * Note that we do the full retry processing here, doing the configured number of + * iterations. + * rgerhards, 2009-05-07 + */ +static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) +{ + int iRetries; + int iSleepPeriod; DEFiRet; ASSERT(pThis != NULL); - pThis->bSuspended = 1; - pThis->ttResumeRtry = tNow + pThis->iResumeInterval; - pThis->iNbrResRtry = 0; /* tell that we did not yet retry to resume */ + + iRetries = 0; + while(pThis->eState == ACT_STATE_RTRY) { + iRet = pThis->pMod->tryResume(pThis->pModData); + if(iRet == RS_RET_OK) { + actionSetState(pThis, ACT_STATE_RDY); + } else if(iRet == RS_RET_SUSPENDED) { + /* max retries reached? */ + if((pThis->iResumeRetryCount != -1 && iRetries >= pThis->iResumeRetryCount)) { + actionSuspend(pThis, ttNow); + } else { + ++pThis->iNbrResRtry; + ++iRetries; + iSleepPeriod = pThis->iResumeInterval; + ttNow += iSleepPeriod; /* not truly exact, but sufficiently... */ + srSleep(iSleepPeriod, 0); + } + } else if(iRet == RS_RET_DISABLE_ACTION) { + actionDisable(pThis); + } + } + + if(pThis->eState == ACT_STATE_RDY) { + pThis->iNbrResRtry = 0; + } RETiRet; } /* try to resume an action -- rgerhards, 2007-08-02 - * returns RS_RET_OK if resumption worked, RS_RET_SUSPEND if the - * action is still suspended. + * changed to new action state engine -- rgerhards, 2009-05-07 */ static rsRetVal actionTryResume(action_t *pThis) { DEFiRet; - time_t ttNow; + time_t ttNow = NO_TIME_PROVIDED; ASSERT(pThis != NULL); - /* for resume handling, we must always obtain a fresh timestamp. We used - * to use the action timestamp, but in this case we will never reach a - * point where a resumption is actually tried, because the action timestamp - * is always in the past. So we can not avoid doing a fresh time() call - * here. -- rgerhards, 2009-03-18 - */ - time(&ttNow); /* cache "now" */ - - /* first check if it is time for a re-try */ - if(ttNow > pThis->ttResumeRtry) { - iRet = pThis->pMod->tryResume(pThis->pModData); - if(iRet == RS_RET_SUSPENDED) { - /* set new tryResume time */ - ++pThis->iNbrResRtry; - /* if we have more than 10 retries, we prolong the - * retry interval. If something is really stalled, it will - * get re-tried only very, very seldom - but that saves - * CPU time. TODO: maybe a config option for that? - * rgerhards, 2007-08-02 - */ - pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); + if(pThis->eState == ACT_STATE_SUSP) { + /* if we are suspended, we need to check if the timeout expired. + * for this handling, we must always obtain a fresh timestamp. We used + * to use the action timestamp, but in this case we will never reach a + * point where a resumption is actually tried, because the action timestamp + * is always in the past. So we can not avoid doing a fresh time() call + * here. -- rgerhards, 2009-03-18 + */ + datetime.GetTime(&ttNow); /* cache "now" */ + if(ttNow > pThis->ttResumeRtry) { + actionSetState(pThis, ACT_STATE_RTRY); /* back to retries */ } - } else { - /* it's too early, we are still suspended --> indicate this */ - iRet = RS_RET_SUSPENDED; } - if(iRet == RS_RET_OK) - actionResume(pThis); + if(pThis->eState == ACT_STATE_RTRY) { + if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */ + datetime.GetTime(&ttNow); + CHKiRet(actionDoRetry(pThis, ttNow)); + } - dbgprintf("actionTryResume: iRet: %d, next retry (if applicable): %u [now %u]\n", - iRet, (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + if(Debug && (pThis->eState == ACT_STATE_RTRY ||pThis->eState == ACT_STATE_SUSP)) { + DBGPRINTF("actionTryResume: action state: %s, next retry (if applicable): %u [now %u]\n", + getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + } +finalize_it: + RETiRet; +} + + +/* prepare an action for performing work. This involves trying to recover it, + * depending on its current state. + * rgerhards, 2009-05-07 + */ +static rsRetVal actionPrepare(action_t *pThis) +{ + DEFiRet; + + assert(pThis != NULL); + CHKiRet(actionTryResume(pThis)); + + /* if we are now ready, we initialize the transaction and advance + * action state accordingly + */ + if(pThis->eState == ACT_STATE_RDY) { + iRet = pThis->pMod->mod.om.beginTransaction(pThis->pModData); + switch(iRet) { + case RS_RET_OK: + actionSetState(pThis, ACT_STATE_ITX); + break; + case RS_RET_SUSPENDED: + actionRetry(pThis); + break; + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + default:FINALIZE; + } + } + +finalize_it: RETiRet; } @@ -402,12 +607,11 @@ rsRetVal actionDbgPrint(action_t *pThis) dbgprintf("\n\tInstance data: 0x%lx\n", (unsigned long) pThis->pModData); dbgprintf("\tRepeatedMsgReduction: %d\n", pThis->f_ReduceRepeated); dbgprintf("\tResume Interval: %d\n", pThis->iResumeInterval); - dbgprintf("\tSuspended: %d", pThis->bSuspended); - if(pThis->bSuspended) { - dbgprintf(" next retry: %u, number retries: %d", (unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry); + if(pThis->eState == ACT_STATE_SUSP) { + dbgprintf("\tresume next retry: %u, number retries: %d", + (unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry); } - dbgprintf("\n"); - dbgprintf("\tDisabled: %d\n", !pThis->bEnabled); + dbgprintf("\tState: %s\n", getActStateName(pThis)); dbgprintf("\tExec only when previous is suspended: %d\n", pThis->bExecWhenPrevSusp); dbgprintf("\n"); @@ -415,112 +619,377 @@ rsRetVal actionDbgPrint(action_t *pThis) } -/* call the DoAction output plugin entry point - * rgerhards, 2008-01-28 +/* prepare the calling parameters for doAction() + * rgerhards, 2009-05-07 */ -#pragma GCC diagnostic ignored "-Wempty-body" -rsRetVal -actionCallDoAction(action_t *pAction, msg_t *pMsg) +static rsRetVal prepareDoActionParams(action_t *pAction, msg_t *pMsg) { - DEFiRet; - int iRetries; int i; - int iArr; - int iSleepPeriod; - int bCallAction; - int iCancelStateSave; - uchar **ppMsgs; /* array of message pointers for doAction */ + DEFiRet; ASSERT(pAction != NULL); - /* create the array for doAction() message pointers */ - if((ppMsgs = calloc(pAction->iNumTpls, sizeof(uchar *))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { switch(pAction->eParamPassing) { case ACT_STRING_PASSING: - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pAction->ppMsgs[i]), &(pAction->lenMsgs[i]))); break; case ACT_ARRAY_PASSING: - CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(ppMsgs[i]))); + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pAction->ppMsgs[i]))); + break; + case ACT_MSG_PASSING: + /* we abuse the uchar* ptr, it now actually is a void*, but we can not + * change that other than by chaning the interface, what we don't like... + */ + pAction->ppMsgs[i] = (uchar*) pMsg; break; default:assert(0); /* software bug if this happens! */ } } - iRetries = 0; - /* We now must guard the output module against execution by multiple threads. The - * plugin interface specifies that output modules must not be thread-safe (except - * if they notify us they are - functionality not yet implemented...). - * rgerhards, 2008-01-30 - */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pAction->mutActExec); - pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - pthread_setcancelstate(iCancelStateSave, NULL); - do { - /* on first invocation, this if should never be true. We just put it at the top - * of the loop so that processing (and code) is simplified. This code is actually - * triggered on the 2nd+ invocation. -- rgerhards, 2008-01-30 - */ - if(iRet == RS_RET_SUSPENDED) { - /* ok, this calls for our retry logic... */ - ++iRetries; - iSleepPeriod = pAction->iResumeInterval; - srSleep(iSleepPeriod, 0); - } - /* first check if we are suspended and, if so, retry */ - if(actionIsSuspended(pAction)) { - iRet = actionTryResume(pAction); - if(iRet == RS_RET_OK) - bCallAction = 1; - else - bCallAction = 0; - } else { - bCallAction = 1; - } - - if(bCallAction) { - /* call configured action */ - iRet = pAction->pMod->mod.om.doAction(ppMsgs, pMsg->msgFlags, pAction->pModData); - if(iRet == RS_RET_SUSPENDED) { - dbgprintf("Action requested to be suspended, done that.\n"); - actionSuspend(pAction, getActNow(pAction)); - } - } - } while(iRet == RS_RET_SUSPENDED && (pAction->iResumeRetryCount == -1 || iRetries < pAction->iResumeRetryCount)); /* do...while! */ +finalize_it: + RETiRet; +} - if(iRet == RS_RET_DISABLE_ACTION) { - dbgprintf("Action requested to be disabled, done that.\n"); - pAction->bEnabled = 0; /* that's it... */ - } - pthread_cleanup_pop(1); /* unlock mutex */ +/* cleanup doAction calling parameters + * rgerhards, 2009-05-07 + */ +static rsRetVal cleanupDoActionParams(action_t *pAction) +{ + int i; + int iArr; + DEFiRet; -finalize_it: - /* cleanup */ + ASSERT(pAction != NULL); for(i = 0 ; i < pAction->iNumTpls ; ++i) { - if(ppMsgs[i] != NULL) { + if(pAction->ppMsgs[i] != NULL) { switch(pAction->eParamPassing) { case ACT_ARRAY_PASSING: iArr = 0; - while(((char **)ppMsgs[i])[iArr] != NULL) - d_free(((char **)ppMsgs[i])[iArr++]); - d_free(ppMsgs[i]); + while(((char **)pAction->ppMsgs[i])[iArr] != NULL) { + d_free(((char **)pAction->ppMsgs[i])[iArr++]); + ((char **)pAction->ppMsgs[i])[iArr++] = NULL; + } + d_free(pAction->ppMsgs[i]); + pAction->ppMsgs[i] = NULL; break; + case ACT_MSG_PASSING: case ACT_STRING_PASSING: - d_free(ppMsgs[i]); break; default: assert(0); } } } - d_free(ppMsgs); - msgDestruct(&pMsg); /* we are now finished with the message */ + + RETiRet; +} + + +/* call the DoAction output plugin entry point + * Performance note: we build the action parameters here in this function. That + * means we do it while we hold the action look, potentially reducing concurrency + * (especially if the action queue is run in DIRECT mode). As an alternative, we + * may generate all params for the batch as whole before aquiring the action. However, + * that requires more memory, for large batches potentially a lot of memory. So for the + * time being, I am doing it here - the performance hit should be very minor and may even + * not be a hit because we may gain CPU cache locality gains with the "fewer memory" + * approach (I'd say that is rater likely). + * rgerhards, 2008-01-28 + */ +rsRetVal +actionCallDoAction(action_t *pThis, msg_t *pMsg) +{ + DEFiRet; + + ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pMsg, msg); + + DBGPRINTF("entering actionCalldoAction(), state: %s\n", getActStateName(pThis)); + CHKiRet(prepareDoActionParams(pThis, pMsg)); + + pThis->bHadAutoCommit = 0; + iRet = pThis->pMod->mod.om.doAction(pThis->ppMsgs, pMsg->msgFlags, pThis->pModData); + switch(iRet) { + case RS_RET_OK: + actionCommitted(pThis); + break; + case RS_RET_DEFER_COMMIT: + /* we are done, action state remains the same */ + break; + case RS_RET_PREVIOUS_COMMITTED: + /* action state remains the same, but we had a commit. */ + pThis->bHadAutoCommit = 1; + break; + case RS_RET_SUSPENDED: + actionRetry(pThis); + break; + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + default:/* permanent failure of this message - no sense in retrying. This is + * not yet handled (but easy TODO) + */ + FINALIZE; + } + iRet = getReturnCode(pThis); + +finalize_it: + cleanupDoActionParams(pThis); /* iRet ignored! */ + + RETiRet; +} + + +/* process a message + * this readies the action and then calls doAction() + * rgerhards, 2008-01-28 + */ +rsRetVal +actionProcessMessage(action_t *pThis, msg_t *pMsg) +{ + DEFiRet; + + ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pMsg, msg); + +RUNLOG_STR("inside actionProcessMsg()"); + CHKiRet(actionPrepare(pThis)); + if(pThis->eState == ACT_STATE_ITX) + CHKiRet(actionCallDoAction(pThis, pMsg)); + + iRet = getReturnCode(pThis); +finalize_it: + RETiRet; +} + + +/* finish processing a batch. Most importantly, that means we commit if we + * need to do so. + * rgerhards, 2008-01-28 + */ +static rsRetVal +finishBatch(action_t *pThis, batch_t *pBatch) +{ + int i; + DEFiRet; + + ASSERT(pThis != NULL); + + if(pThis->eState == ACT_STATE_RDY) + FINALIZE; /* nothing to do */ + + CHKiRet(actionPrepare(pThis)); + if(pThis->eState == ACT_STATE_ITX) { + iRet = pThis->pMod->mod.om.endTransaction(pThis->pModData); + switch(iRet) { + case RS_RET_OK: + actionCommitted(pThis); + /* flag messages as committed */ + for(i = 0 ; i < pBatch->nElem ; ++i) { + pBatch->pElem[i].state = BATCH_STATE_COMM; + } + break; + case RS_RET_SUSPENDED: + actionRetry(pThis); + break; + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + case RS_RET_DEFER_COMMIT: + DBGPRINTF("output plugin error: endTransaction() returns RS_RET_DEFER_COMMIT " + "- ignored\n"); + actionCommitted(pThis); + break; + case RS_RET_PREVIOUS_COMMITTED: + DBGPRINTF("output plugin error: endTransaction() returns RS_RET_PREVIOUS_COMMITTED " + "- ignored\n"); + actionCommitted(pThis); + break; + default:/* permanent failure of this message - no sense in retrying. This is + * not yet handled (but easy TODO) + */ + FINALIZE; + } + } + iRet = getReturnCode(pThis); + +finalize_it: + RETiRet; +} + + +/* try to submit a partial batch of elements. + * rgerhards, 2009-05-12 + */ +static rsRetVal +tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem, int *pbShutdownImmediate) +{ + int i; + int iElemProcessed; + int iCommittedUpTo; + msg_t *pMsg; + rsRetVal localRet; + DEFiRet; + + assert(pBatch != NULL); + assert(pnElem != NULL); + + i = pBatch->iDoneUpTo; /* all messages below that index are processed */ + iElemProcessed = 0; + iCommittedUpTo = i; + while(iElemProcessed <= *pnElem && i < pBatch->nElem) { + if(*pbShutdownImmediate) + ABORT_FINALIZE(RS_RET_FORCE_TERM); + pMsg = (msg_t*) pBatch->pElem[i].pUsrp; + if(pBatch->pElem[i].state != BATCH_STATE_DISC) { + localRet = actionProcessMessage(pAction, pMsg); + DBGPRINTF("action call returned %d\n", localRet); + if(localRet == RS_RET_OK) { + /* mark messages as committed */ + while(iCommittedUpTo <= i) { + pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; + } + } else if(localRet == RS_RET_PREVIOUS_COMMITTED) { + /* mark messages as committed */ + while(iCommittedUpTo < i) { + pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; + } + pBatch->pElem[i].state = BATCH_STATE_SUB; + } else if(localRet == RS_RET_DEFER_COMMIT) { + pBatch->pElem[i].state = BATCH_STATE_SUB; + } else if(localRet == RS_RET_DISCARDMSG) { + pBatch->pElem[i].state = BATCH_STATE_DISC; + } else { + dbgprintf("tryDoAction: unexpected error code %d, finalizing\n", localRet); + iRet = localRet; + FINALIZE; + } + } + ++i; + ++iElemProcessed; + } + +finalize_it: + if(pBatch->nElem == 1 && pBatch->pElem[0].state == BATCH_STATE_DISC) { + iRet = RS_RET_DISCARDMSG; + } else if(pBatch->iDoneUpTo != iCommittedUpTo) { + *pnElem += iCommittedUpTo - pBatch->iDoneUpTo; + pBatch->iDoneUpTo = iCommittedUpTo; + } + RETiRet; +} + + +/* submit a batch for actual action processing. + * The first nElem elements are processed. This function calls itself + * recursively if it needs to handle errors. + * rgerhards, 2009-05-12 + */ +static rsRetVal +submitBatch(action_t *pAction, batch_t *pBatch, int nElem, int *pbShutdownImmediate) +{ + int i; + int bDone; + rsRetVal localRet; + DEFiRet; + + assert(pBatch != NULL); + + bDone = 0; + do { + localRet = tryDoAction(pAction, pBatch, &nElem, pbShutdownImmediate); + if(localRet == RS_RET_FORCE_TERM) + FINALIZE; + if( localRet == RS_RET_OK + || localRet == RS_RET_PREVIOUS_COMMITTED + || localRet == RS_RET_DEFER_COMMIT) { + /* try commit transaction, once done, we can simply do so as if + * that return state was returned from tryDoAction(). + */ + localRet = finishBatch(pAction, pBatch); // TODO: careful, do we need the elem counter? + } + + if( localRet == RS_RET_OK + || localRet == RS_RET_PREVIOUS_COMMITTED + || localRet == RS_RET_DEFER_COMMIT) { + bDone = 1; + } else if(localRet == RS_RET_DISCARDMSG) { + iRet = RS_RET_DISCARDMSG; /* TODO: verify this sequence -- rgerhards, 2009-07-30 */ + bDone = 1; + } else if(localRet == RS_RET_SUSPENDED) { + ; /* do nothing, this will retry the full batch */ + } else if(localRet == RS_RET_ACTION_FAILED) { + /* in this case, the whole batch can not be processed */ + for(i = 0 ; i < nElem ; ++i) { + pBatch->pElem[pBatch->iDoneUpTo++].state = BATCH_STATE_BAD; + } + bDone = 1; + } else { + if(nElem == 1) { + pBatch->pElem[pBatch->iDoneUpTo++].state = BATCH_STATE_BAD; +// TODO: This is a mark, remove when no longer needed - Here was the bug, postincrement needs to be used, not preinc + bDone = 1; + } else { + /* retry with half as much. Depth is log_2 batchsize, so recursion is not too deep */ + submitBatch(pAction, pBatch, nElem / 2, pbShutdownImmediate); + submitBatch(pAction, pBatch, nElem - (nElem / 2), pbShutdownImmediate); + bDone = 1; + } + } + } while(!bDone && !*pbShutdownImmediate); /* do .. while()! */ + + if(*pbShutdownImmediate) + ABORT_FINALIZE(RS_RET_FORCE_TERM); + +finalize_it: + RETiRet; +} + + +/* receive a batch and process it. This includes retry handling. + * rgerhards, 2009-05-12 + */ +static rsRetVal +processAction(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) +{ + DEFiRet; + + assert(pBatch != NULL); + pBatch->iDoneUpTo = 0; + CHKiRet(submitBatch(pAction, pBatch, pBatch->nElem, pbShutdownImmediate)); + iRet = finishBatch(pAction, pBatch); + +finalize_it: + RETiRet; +} + + +#pragma GCC diagnostic ignored "-Wempty-body" +/* receive an array of to-process user pointers and submit them + * for processing. + * rgerhards, 2009-04-22 + */ +static rsRetVal +processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) +{ + DEFiRet; + + assert(pBatch != NULL); + + /* We now must guard the output module against execution by multiple threads. The + * plugin interface specifies that output modules must not be thread-safe (except + * if they notify us they are - functionality not yet implemented...). + * rgerhards, 2008-01-30 + */ + d_pthread_mutex_lock(&pAction->mutActExec); + pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); + + iRet = processAction(pAction, pBatch, pbShutdownImmediate); + + pthread_cleanup_pop(1); /* unlock mutex */ RETiRet; } @@ -537,7 +1006,6 @@ rsRetVal actionCallHUPHdlr(action_t *pAction) { DEFiRet; - int iCancelStateSave; ASSERT(pAction != NULL); DBGPRINTF("Action %p checks HUP hdlr: %p\n", pAction, pAction->pMod->doHUP); @@ -546,10 +1014,8 @@ actionCallHUPHdlr(action_t *pAction) FINALIZE; /* no HUP handler, so we are done ;) */ } - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); d_pthread_mutex_lock(&pAction->mutActExec); pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - pthread_setcancelstate(iCancelStateSave, NULL); CHKiRet(pAction->pMod->doHUP(pAction->pModData)); pthread_cleanup_pop(1); /* unlock mutex */ @@ -569,16 +1035,16 @@ static rsRetVal setActionQueType(void __attribute__((unused)) *pVal, uchar *pszT if (!strcasecmp((char *) pszType, "fixedarray")) { ActionQueType = QUEUETYPE_FIXED_ARRAY; - dbgprintf("action queue type set to FIXED_ARRAY\n"); + DBGPRINTF("action queue type set to FIXED_ARRAY\n"); } else if (!strcasecmp((char *) pszType, "linkedlist")) { ActionQueType = QUEUETYPE_LINKEDLIST; - dbgprintf("action queue type set to LINKEDLIST\n"); + DBGPRINTF("action queue type set to LINKEDLIST\n"); } else if (!strcasecmp((char *) pszType, "disk")) { ActionQueType = QUEUETYPE_DISK; - dbgprintf("action queue type set to DISK\n"); + DBGPRINTF("action queue type set to DISK\n"); } else if (!strcasecmp((char *) pszType, "direct")) { ActionQueType = QUEUETYPE_DIRECT; - dbgprintf("action queue type set to DIRECT (no queueing at all)\n"); + DBGPRINTF("action queue type set to DIRECT (no queueing at all)\n"); } else { errmsg.LogError(0, RS_RET_INVALID_PARAMS, "unknown actionqueue parameter: %s", (char *) pszType); iRet = RS_RET_INVALID_PARAMS; @@ -624,15 +1090,15 @@ actionWriteToAction(action_t *pAction) /* we need to care about multiple occurences */ if( pAction->iExecEveryNthOccurTO > 0 && (getActNow(pAction) - pAction->tLastOccur) > pAction->iExecEveryNthOccurTO) { - dbgprintf("n-th occurence handling timed out (%d sec), restarting from 0\n", + DBGPRINTF("n-th occurence handling timed out (%d sec), restarting from 0\n", (int) (getActNow(pAction) - pAction->tLastOccur)); pAction->iNbrNoExec = 0; pAction->tLastOccur = getActNow(pAction); } if(pAction->iNbrNoExec < pAction->iExecEveryNthOccur - 1) { ++pAction->iNbrNoExec; - dbgprintf("action %p passed %d times to execution - less than neded - discarding\n", - pAction, pAction->iNbrNoExec); + DBGPRINTF("action %p passed %d times to execution - less than neded - discarding\n", + pAction, pAction->iNbrNoExec); FINALIZE; } else { pAction->iNbrNoExec = 0; /* we execute the action now, so the number of no execs is down to */ @@ -649,36 +1115,34 @@ actionWriteToAction(action_t *pAction) */ if(pAction->f_prevcount > 1) { msg_t *pMsg; + size_t lenRepMsg; uchar szRepMsg[1024]; if((pMsg = MsgDup(pAction->f_pMsg)) == NULL) { /* it failed - nothing we can do against it... */ - dbgprintf("Message duplication failed, dropping repeat message.\n"); + DBGPRINTF("Message duplication failed, dropping repeat message.\n"); ABORT_FINALIZE(RS_RET_ERR); } if(pAction->bRepMsgHasMsg == 0) { /* old format repeat message? */ - snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", + lenRepMsg = snprintf((char*)szRepMsg, sizeof(szRepMsg), " last message repeated %d times", pAction->f_prevcount); } else { - snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", + lenRepMsg = snprintf((char*)szRepMsg, sizeof(szRepMsg), " message repeated %d times: [%.800s]", pAction->f_prevcount, getMSG(pAction->f_pMsg)); } - /* We now need to update the other message properties. - * ... RAWMSG is a problem ... Please note that digital + /* We now need to update the other message properties. Please note that digital * signatures inside the message are also invalidated. */ datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); - MsgSetMSG(pMsg, (char*)szRepMsg); - MsgSetRawMsg(pMsg, (char*)szRepMsg); - + MsgReplaceMSG(pMsg, szRepMsg, lenRepMsg); pMsgSave = pAction->f_pMsg; /* save message pointer for later restoration */ pAction->f_pMsg = pMsg; /* use the new msg (pointer will be restored below) */ } - dbgprintf("Called action, logging to %s\n", module.GetStateName(pAction->pMod)); + DBGPRINTF("Called action, logging to %s\n", module.GetStateName(pAction->pMod)); /* now check if we need to drop the message because otherwise the action would be too * frequently called. -- rgerhards, 2008-04-08 @@ -686,10 +1150,10 @@ actionWriteToAction(action_t *pAction) * a purely logical point of view. However, if safes us to check the system time in * (those common) cases where ExecOnceInterval is not used. -- rgerhards, 2008-09-16 */ - if(pAction->f_time != 0 && pAction->iSecsExecOnceInterval > 0 && + if(pAction->iSecsExecOnceInterval > 0 && pAction->iSecsExecOnceInterval + pAction->tLastExec > getActNow(pAction)) { /* in this case we need to discard the message - its not yet time to exec the action */ - dbgprintf("action not yet ready again to be executed, onceInterval %d, tCurr %d, tNext %d\n", + DBGPRINTF("action not yet ready again to be executed, onceInterval %d, tCurr %d, tNext %d\n", (int) pAction->iSecsExecOnceInterval, (int) getActNow(pAction), (int) (pAction->iSecsExecOnceInterval + pAction->tLastExec)); pAction->tLastExec = getActNow(pAction); /* re-init time flags */ @@ -697,6 +1161,7 @@ actionWriteToAction(action_t *pAction) } /* we use reception time, not dequeue time - this is considered more appropriate and also faster ;) -- rgerhards, 2008-09-17 */ + pAction->tLastExec = getActNow(pAction); /* re-init time flags */ pAction->f_time = pAction->f_pMsg->ttGenTime; /* When we reach this point, we have a valid, non-disabled action. @@ -726,62 +1191,31 @@ finalize_it: } -/* call the configured action. Does all necessary housekeeping. - * rgerhards, 2007-08-01 - * FYI: currently, this function is only called from the queue - * consumer. So we (conceptually) run detached from the input - * threads (which also means we may run much later than when the - * message was generated). +/* helper to actonCallAction, mostly needed because of this damn + * pthread_cleanup_push() POSIX macro... */ -#pragma GCC diagnostic ignored "-Wempty-body" -rsRetVal -actionCallAction(action_t *pAction, msg_t *pMsg) +static rsRetVal +doActionCallAction(action_t *pAction, msg_t *pMsg) { DEFiRet; - int iCancelStateSave; - - ISOBJ_TYPE_assert(pMsg, msg); - ASSERT(pAction != NULL); - - /* Make sure nodbody else modifies/uses this action object. Right now, this - * is important because of "message repeated n times" processing and potentially - * multiple worker threads. -- rgerhards, 2007-12-11 - */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - LockObj(pAction); - pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); - pthread_setcancelstate(iCancelStateSave, NULL); - - /* first, we need to check if this is a disabled - * entry. If so, we must not further process it. - * rgerhards 2005-09-26 - * In the future, disabled modules may be re-probed from time - * to time. They are in a perfectly legal state, except that the - * doAction method indicated that it wanted to be disabled - but - * we do not consider this is a solution for eternity... So we - * should check from time to time if affairs have improved. - * rgerhards, 2007-07-24 - */ - if(pAction->bEnabled == 0) { - ABORT_FINALIZE(RS_RET_OK); - } pAction->tActNow = -1; /* we do not yet know our current time (clear prev. value) */ /* don't output marks to recently written outputs */ - if((pMsg->msgFlags & MARK) && (getActNow(pAction) - pAction->f_time) < MarkInterval / 2) { + if(pAction->bWriteAllMarkMsgs == FALSE + && (pMsg->msgFlags & MARK) && (getActNow(pAction) - pAction->f_time) < MarkInterval / 2) { ABORT_FINALIZE(RS_RET_OK); } /* suppress duplicate messages */ if ((pAction->f_ReduceRepeated == 1) && pAction->f_pMsg != NULL && (pMsg->msgFlags & MARK) == 0 && getMSGLen(pMsg) == getMSGLen(pAction->f_pMsg) && - !strcmp(getMSG(pMsg), getMSG(pAction->f_pMsg)) && + !ustrcmp(getMSG(pMsg), getMSG(pAction->f_pMsg)) && !strcmp(getHOSTNAME(pMsg), getHOSTNAME(pAction->f_pMsg)) && - !strcmp(getPROCID(pMsg), getPROCID(pAction->f_pMsg)) && - !strcmp(getAPPNAME(pMsg), getAPPNAME(pAction->f_pMsg))) { + !strcmp(getPROCID(pMsg, LOCK_MUTEX), getPROCID(pAction->f_pMsg, LOCK_MUTEX)) && + !strcmp(getAPPNAME(pMsg, LOCK_MUTEX), getAPPNAME(pAction->f_pMsg, LOCK_MUTEX))) { pAction->f_prevcount++; - dbgprintf("msg repeated %d times, %ld sec of %d.\n", + DBGPRINTF("msg repeated %d times, %ld sec of %d.\n", pAction->f_prevcount, (long) getActNow(pAction) - pAction->f_time, repeatinterval[pAction->f_repeatcount]); /* use current message, so we have the new timestamp (means we need to discard previous one) */ @@ -812,51 +1246,42 @@ actionCallAction(action_t *pAction, msg_t *pMsg) } finalize_it: - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - UnlockObj(pAction); - pthread_cleanup_pop(0); /* remove mutex cleanup handler */ - pthread_setcancelstate(iCancelStateSave, NULL); RETiRet; } -#pragma GCC diagnostic warning "-Wempty-body" -/* add our cfsysline handlers - * rgerhards, 2008-01-28 +/* call the configured action. Does all necessary housekeeping. + * rgerhards, 2007-08-01 + * FYI: currently, this function is only called from the queue + * consumer. So we (conceptually) run detached from the input + * threads (which also means we may run much later than when the + * message was generated). */ +#pragma GCC diagnostic ignored "-Wempty-body" rsRetVal -actionAddCfSysLineHdrl(void) +actionCallAction(action_t *pAction, msg_t *pMsg) { DEFiRet; - CHKiRet(regCfSysLineHdlr((uchar *)"actionname", 0, eCmdHdlrGetWord, NULL, &pszActionName, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszActionQFName, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesize", 0, eCmdHdlrInt, NULL, &iActionQueueSize, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iActionQueMaxDiskSpace, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iActionQHighWtrMark, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuelowwatermark", 0, eCmdHdlrInt, NULL, &iActionQLowWtrMark, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardmark", 0, eCmdHdlrInt, NULL, &iActionQDiscardMark, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardseverity", 0, eCmdHdlrInt, NULL, &iActionQDiscardSeverity, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iActionQPersistUpdCnt, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetype", 0, eCmdHdlrGetWord, setActionQueType, NULL, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iActionQueueNumWorkers, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoQShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutactioncompletion", 0, eCmdHdlrInt, NULL, &iActionQtoActShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutenqueue", 0, eCmdHdlrInt, NULL, &iActionQtoEnq, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkertimeoutthreadshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoWrkShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreadminimummessages", 0, eCmdHdlrInt, NULL, &iActionQWrkMinMsgs, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxfilesize", 0, eCmdHdlrSize, NULL, &iActionQueMaxFileSize, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesaveonshutdown", 0, eCmdHdlrBinary, NULL, &bActionQSaveOnShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeueslowdown", 0, eCmdHdlrInt, NULL, &iActionQueueDeqSlowdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimebegin", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinFromHr, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinToHr, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtime", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccur, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtimetimeout", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccurTO, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgcontainsoriginalmsg", 0, eCmdHdlrBinary, NULL, &bActionRepMsgHasMsg, NULL)); - -finalize_it: + ISOBJ_TYPE_assert(pMsg, msg); + ASSERT(pAction != NULL); + + /* We need to lock the mutex only for repeated line processing. + * rgerhards, 2009-06-19 + */ + //if(pAction->f_ReduceRepeated == 1) { + LockObj(pAction); + pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); + iRet = doActionCallAction(pAction, pMsg); + UnlockObj(pAction); + pthread_cleanup_pop(0); /* remove mutex cleanup handler */ + //} else { + //iRet = doActionCallAction(pAction, pMsg); + //} + RETiRet; } +#pragma GCC diagnostic warning "-Wempty-body" /* add an Action to the current selector @@ -877,13 +1302,15 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques assert(ppAction != NULL); assert(pMod != NULL); assert(pOMSR != NULL); - dbgprintf("Module %s processed this config line.\n", module.GetName(pMod)); + DBGPRINTF("Module %s processed this config line.\n", module.GetName(pMod)); CHKiRet(actionConstruct(&pAction)); /* create action object first */ pAction->pMod = pMod; pAction->pModData = pModData; pAction->pszName = pszActionName; pszActionName = NULL; /* free again! */ + pAction->bWriteAllMarkMsgs = bActionWriteAllMarkMsgs; + bActionWriteAllMarkMsgs = FALSE; /* reset */ pAction->bExecWhenPrevSusp = bActExecWhenPrevSusp; pAction->iSecsExecOnceInterval = iActExecOnceInterval; pAction->iExecEveryNthOccur = iActExecEveryNthOccur; @@ -901,16 +1328,15 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques */ if(pAction->iNumTpls > 0) { /* we first need to create the template pointer array */ - if((pAction->ppTpl = calloc(pAction->iNumTpls, sizeof(struct template *))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pAction->ppTpl = (struct template **)calloc(pAction->iNumTpls, sizeof(struct template *))); + CHKmalloc(pAction->ppMsgs = (uchar**) calloc(pAction->iNumTpls, sizeof(uchar *))); + CHKmalloc(pAction->lenMsgs = (size_t*) calloc(pAction->iNumTpls, sizeof(size_t))); } for(i = 0 ; i < pAction->iNumTpls ; ++i) { CHKiRet(OMSRgetEntry(pOMSR, i, &pTplName, &iTplOpts)); - /* Ok, we got everything, so it now is time to look up the - * template (Hint: templates MUST be defined before they are - * used!) + /* Ok, we got everything, so it now is time to look up the template + * (Hint: templates MUST be defined before they are used!) */ if((pAction->ppTpl[i] = tplFind((char*)pTplName, strlen((char*)pTplName))) == NULL) { snprintf(errMsg, sizeof(errMsg) / sizeof(char), @@ -932,11 +1358,13 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques /* set parameter-passing mode */ if(iTplOpts & OMSR_TPL_AS_ARRAY) { pAction->eParamPassing = ACT_ARRAY_PASSING; + } else if(iTplOpts & OMSR_TPL_AS_MSG) { + pAction->eParamPassing = ACT_MSG_PASSING; } else { pAction->eParamPassing = ACT_STRING_PASSING; } - dbgprintf("template: '%s' assigned\n", pTplName); + DBGPRINTF("template: '%s' assigned\n", pTplName); } pAction->pMod = pMod; @@ -945,13 +1373,13 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques if(pMod->isCompatibleWithFeature(sFEATURERepeatedMsgReduction) == RS_RET_OK) pAction->f_ReduceRepeated = bReduceRepeatMsgs; else { - dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); + DBGPRINTF("module is incompatible with RepeatedMsgReduction - turned off\n"); pAction->f_ReduceRepeated = 0; } - pAction->bEnabled = 1; /* action is enabled */ + pAction->eState = ACT_STATE_RDY; /* action is enabled */ if(bSuspended) - actionSuspend(pAction, time(NULL)); /* "good" time call, only during init and unavoidable */ + actionSuspend(pAction, datetime.GetTime(NULL)); /* "good" time call, only during init and unavoidable */ CHKiRet(actionConstructFinalize(pAction)); @@ -973,6 +1401,17 @@ finalize_it: } +/* Reset config variables to default values. + * rgerhards, 2009-11-12 + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + iActExecOnceInterval = 0; + return RS_RET_OK; +} + + /* TODO: we are not yet a real object, the ClassInit here just looks like it is.. */ rsRetVal actionClassInit(void) @@ -984,6 +1423,36 @@ rsRetVal actionClassInit(void) CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionname", 0, eCmdHdlrGetWord, NULL, &pszActionName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszActionQFName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesize", 0, eCmdHdlrInt, NULL, &iActionQueueSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionwriteallmarkmessages", 0, eCmdHdlrBinary, NULL, &bActionWriteAllMarkMsgs, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuebatchsize", 0, eCmdHdlrInt, NULL, &iActionQueueDeqBatchSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iActionQueMaxDiskSpace, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iActionQHighWtrMark, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuelowwatermark", 0, eCmdHdlrInt, NULL, &iActionQLowWtrMark, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardmark", 0, eCmdHdlrInt, NULL, &iActionQDiscardMark, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardseverity", 0, eCmdHdlrInt, NULL, &iActionQDiscardSeverity, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iActionQPersistUpdCnt, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bActionQSyncQeueFiles, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetype", 0, eCmdHdlrGetWord, setActionQueType, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iActionQueueNumWorkers, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoQShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutactioncompletion", 0, eCmdHdlrInt, NULL, &iActionQtoActShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutenqueue", 0, eCmdHdlrInt, NULL, &iActionQtoEnq, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkertimeoutthreadshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoWrkShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreadminimummessages", 0, eCmdHdlrInt, NULL, &iActionQWrkMinMsgs, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxfilesize", 0, eCmdHdlrSize, NULL, &iActionQueMaxFileSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesaveonshutdown", 0, eCmdHdlrBinary, NULL, &bActionQSaveOnShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeueslowdown", 0, eCmdHdlrInt, NULL, &iActionQueueDeqSlowdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimebegin", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinFromHr, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinToHr, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtime", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccur, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtimetimeout", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccurTO, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyonceeveryinterval", 0, eCmdHdlrInt, NULL, &iActExecOnceInterval, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgcontainsoriginalmsg", 0, eCmdHdlrBinary, NULL, &bActionRepMsgHasMsg, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); + finalize_it: RETiRet; } @@ -36,6 +36,15 @@ extern int glbliActionResumeRetryCount; +typedef enum { + ACT_STATE_DIED = 0, /* action permanently failed and now disabled - MUST BE ZEO! */ + ACT_STATE_RDY = 1, /* action ready, waiting for new transaction */ + ACT_STATE_ITX = 2, /* transaction active, waiting for new data or commit */ + ACT_STATE_COMM = 3, /* transaction finished (a transient state) */ + ACT_STATE_RTRY = 4, /* failure occured, trying to restablish ready state */ + ACT_STATE_SUSP = 5 /* suspended due to failure (return fail until timeout expired) */ +} action_state_t; + /* the following struct defines the action object data structure */ struct action_s { @@ -43,10 +52,11 @@ struct action_s { time_t tActNow; /* the current time for an action execution. Initially set to -1 and populated on an as-needed basis. This is a performance optimization. */ time_t tLastExec; /* time this action was last executed */ - int bExecWhenPrevSusp;/* execute only when previous action is suspended? */ + sbool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ + sbool bWriteAllMarkMsgs;/* should all mark msgs be written (not matter how recent the action was executed)? */ int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ - short bEnabled; /* is the related action enabled (1) or disabled (0)? */ - short bSuspended; /* is the related action temporarily suspended? */ + action_state_t eState; /* current state of action */ + int bHadAutoCommit; /* did an auto-commit happen during doAction()? */ time_t ttResumeRtry; /* when is it time to retry the resume? */ int iResumeInterval;/* resume interval for this action */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ @@ -57,23 +67,25 @@ struct action_s { time_t tLastOccur; /* time last occurence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ - short bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ + sbool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ - enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1 } + enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1, ACT_MSG_PASSING } eParamPassing; /* mode of parameter passing to action */ int iNumTpls; /* number of array entries for template element below */ struct template **ppTpl;/* array of template to use - strings must be passed to doAction * in this order. */ - struct msg* f_pMsg; /* pointer to the message (this will replace the other vars with msg + msg_t *f_pMsg; /* pointer to the message (this will replace the other vars with msg * content later). This is preserved after the message has been * processed - it is also used to detect duplicates. */ qqueue_t *pQueue; /* action queue */ SYNC_OBJ_TOOL; /* required for mutex support */ - uchar *pszName; /* action name (for documentation) */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ + uchar *pszName; /* action name (for documentation) */ + uchar **ppMsgs; /* pointer to action-calling parameters (kept in structure to save alloc() time!) */ + size_t *lenMsgs; /* length of message in ppMsgs */ }; typedef struct action_s action_t; @@ -83,7 +95,6 @@ typedef struct action_s action_t; rsRetVal actionConstruct(action_t **ppThis); rsRetVal actionConstructFinalize(action_t *pThis); rsRetVal actionDestruct(action_t *pThis); -rsRetVal actionAddCfSysLineHdrl(void); rsRetVal actionDbgPrint(action_t *pThis); rsRetVal actionSetGlobalResumeInterval(int iNewVal); rsRetVal actionDoAction(action_t *pAction); @@ -19,7 +19,7 @@ fi (cd $srcdir && autoreconf --verbose --force --install) || exit 1 -conf_flags="--enable-shave --cache-file=config.cache" +conf_flags="--cache-file=config.cache" if test x$NOCONFIGURE = x; then echo Running $srcdir/configure $conf_flags "$@" ... diff --git a/configure.ac b/configure.ac index 66f1cab1..24e96965 100644 --- a/configure.ac +++ b/configure.ac @@ -2,14 +2,23 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.3.1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.5.2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE + +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) + AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) AC_GNU_SOURCE +# check for Java compiler +AC_CHECK_PROG(HAVE_JAVAC, [javac], [yes]) +if test x"$HAVE_JAVAC" = x""; then + AC_MSG_WARN([no javac found, disabling features depending on it]) +fi + # Checks for programs. AC_PROG_CC AM_PROG_CC_C_O @@ -63,7 +72,7 @@ AC_SUBST(DL_LIBS) AC_HEADER_RESOLV AC_HEADER_STDC AC_HEADER_SYS_WAIT -AC_CHECK_HEADERS([arpa/inet.h libgen.h fcntl.h locale.h netdb.h netinet/in.h paths.h stddef.h stdlib.h string.h sys/file.h sys/ioctl.h sys/param.h sys/socket.h sys/time.h sys/stat.h syslog.h unistd.h utmp.h sys/epoll.h]) +AC_CHECK_HEADERS([arpa/inet.h libgen.h malloc.h fcntl.h locale.h netdb.h netinet/in.h paths.h stddef.h stdlib.h string.h sys/file.h sys/ioctl.h sys/param.h sys/socket.h sys/time.h sys/stat.h syslog.h unistd.h utmp.h utmpx.h sys/epoll.h sys/prctl.h]) # Checks for typedefs, structures, and compiler characteristics. AC_C_CONST @@ -98,7 +107,7 @@ AC_TYPE_SIGNAL AC_FUNC_STAT AC_FUNC_STRERROR_R AC_FUNC_VPRINTF -AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline]) +AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r getline malloc_trim prctl epoll_create epoll_create1 fdatasync]) # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) @@ -119,6 +128,7 @@ AC_TRY_COMPILE([ # check for availability of atomic operations RS_ATOMIC_OPERATIONS +RS_ATOMIC_OPERATIONS_64BIT # Additional module directories @@ -176,6 +186,7 @@ if test "$enable_regexp" = "yes"; then fi + # zlib compression AC_ARG_ENABLE(zlib, [AS_HELP_STRING([--enable-zlib],[Enable zlib compression support @<:@default=yes@:>@])], @@ -186,6 +197,7 @@ AC_ARG_ENABLE(zlib, esac], [enable_zlib=yes] ) +AM_CONDITIONAL(ENABLE_ZLIB, test x$enable_zlib = xyes) if test "$enable_zlib" = "yes"; then AC_CHECK_HEADER(zlib.h, [zlib_header="yes"], [zlib_header="no" enable_zlib="false"]) if test "$zlib_header" = "yes"; then @@ -232,7 +244,7 @@ AC_ARG_ENABLE(pthreads, ) if test "x$enable_pthreads" = "xno"; then - AC_MSG_ERROR(rsyslog v3 does no longer support single threading mode -- use a previous version for that); + AC_MSG_ERROR(rsyslog v3+ does no longer support single threading mode -- use a previous version for that); fi if test "x$enable_pthreads" != "xno"; then @@ -335,6 +347,21 @@ AC_ARG_ENABLE([fsstnd], ]) +# support for unlimited select() syscall +AC_ARG_ENABLE(unlimited_select, + [AS_HELP_STRING([--enable-unlimited-select],[Enable unlimited select() syscall @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_unlimited_select="yes" ;; + no) enable_unlimited_select="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-unlimited-select) ;; + esac], + [enable_unlimited_select="no"] +) +if test "$enable_unlimited_select" = "yes"; then + AC_DEFINE(USE_UNLIMITED_SELECT, 1, [If defined, the select() syscall won't be limited to a particular number of file descriptors.]) +fi + + # debug AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], @@ -383,6 +410,21 @@ if test "$enable_valgrind" = "yes"; then fi +# memcheck +AC_ARG_ENABLE(memcheck, + [AS_HELP_STRING([--enable-memcheck],[Enable extended memory check support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_memcheck="yes" ;; + no) enable_memcheck="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-memcheck) ;; + esac], + [enable_memcheck="no"] +) +if test "$enable_memcheck" = "yes"; then + AC_DEFINE(MEMCHECK, 1, [Defined if memcheck support settings are to be enabled (e.g. prevents dlclose()).]) +fi + + # compile diagnostic tools (small helpers usually not needed) AC_ARG_ENABLE(diagtools, [AS_HELP_STRING([--enable-diagtools],[Enable diagnostic tools @<:@default=no@:>@])], @@ -414,7 +456,7 @@ if test "x$enable_mysql" = "xyes"; then [yes],,, ) if test "x${HAVE_MYSQL_CONFIG}" != "xyes"; then - AC_MSG_FAILURE([mysql_config not found in PATH]) + AC_MSG_FAILURE([mysql_config not found in PATH - usually a package named mysql-dev, libmysql-dev or similar, is missing - install it to fix this issue]) fi AC_CHECK_LIB( [mysqlclient], @@ -573,7 +615,7 @@ AC_ARG_ENABLE(gnutls, [enable_gnutls=no] ) if test "x$enable_gnutls" = "xyes"; then - PKG_CHECK_MODULES(GNUTLS, gnutls >= 2.0.0) + PKG_CHECK_MODULES(GNUTLS, gnutls >= 1.4.0) fi AM_CONDITIONAL(ENABLE_GNUTLS, test x$enable_gnutls = xyes) AC_SUBST(GNUTLS_CFLAGS) @@ -678,6 +720,25 @@ AC_SUBST(LIBLOGGING_CFLAGS) AC_SUBST(LIBLOGGING_LIBS) +# enable/disable the testbench (e.g. because some important parts +# are missing) +AC_ARG_ENABLE(testbench, + [AS_HELP_STRING([--enable-testbench],[file input module enabled @<:@default=yes@:>@])], + [case "${enableval}" in + yes) enable_testbench="yes" ;; + no) enable_testbench="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-testbench) ;; + esac], + [enable_testbench=yes] +) +if test "$enable_testbench" = "yes"; then + if test x$HAVE_JAVAC = x; then + enable_testbench='no' + fi +fi +AM_CONDITIONAL(ENABLE_TESTBENCH, test x$enable_testbench = xyes) + + # settings for the file input module AC_ARG_ENABLE(imfile, [AS_HELP_STRING([--enable-imfile],[file input module enabled @<:@default=no@:>@])], @@ -704,6 +765,36 @@ AC_ARG_ENABLE(omprog, AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) +# settings for omudpspoof +AC_ARG_ENABLE(omudpspoof, + [AS_HELP_STRING([--enable-omudpspoof],[Compiles omudpspoof module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omudpspoof="yes" ;; + no) enable_omudpspoof="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omudpspoof) ;; + esac], + [enable_omudpspoof=no] +) + +if test "x$enable_omudpspoof" = "xyes"; then + AC_CHECK_HEADERS( + [libnet.h],, + [AC_MSG_FAILURE([libnet is missing])] + ) + AC_CHECK_LIB( + [net], + [libnet_init], + [UDPSPOOF_CFLAGS="" + UDPSPOOF_LIBS="-lnet" + ], + [AC_MSG_FAILURE([libnet is missing])] + ) +fi +AM_CONDITIONAL(ENABLE_OMUDPSPOOF, test x$enable_omudpspoof = xyes) +AC_SUBST(UDPSPOOF_CFLAGS) +AC_SUBST(UDPSPOOF_LIBS) + + # settings for omstdout AC_ARG_ENABLE(omstdout, [AS_HELP_STRING([--enable-omstdout],[Compiles stdout module @<:@default=no@:>@])], @@ -717,6 +808,68 @@ AC_ARG_ENABLE(omstdout, AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) +# settings for omruleset +AC_ARG_ENABLE(omruleset, + [AS_HELP_STRING([--enable-omruleset],[Compiles ruleset forwarding module @<:@default=yes@:>@])], + [case "${enableval}" in + yes) enable_omruleset="yes" ;; + no) enable_omruleset="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omruleset) ;; + esac], + [enable_omruleset=yes] +) +AM_CONDITIONAL(ENABLE_OMRULESET, test x$enable_omruleset = xyes) + + +# settings for omdbalerting +AC_ARG_ENABLE(omdbalerting, + [AS_HELP_STRING([--enable-omdbalerting],[Compiles omdbalerting module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omdbalerting="yes" ;; + no) enable_omdbalerting="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omdbalerting) ;; + esac], + [enable_omdbalerting=no] +) +AM_CONDITIONAL(ENABLE_OMDBALERTING, test x$enable_omdbalerting = xyes) + + + +# building the GUI (mostly for diagnostic reasons) +AC_ARG_ENABLE(gui, + [AS_HELP_STRING([--enable-gui],[Enable GUI programs @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_gui="yes" ;; + no) enable_gui="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-gui) ;; + esac], + [enable_gui=no] +) +if test "x$enable_gui" = "xyes"; then + if test x$HAVE_JAVAC = x; then + AC_MSG_ERROR([GUI components need Java, but Java development system is not installed on this system]) + fi +fi +AM_CONDITIONAL(ENABLE_GUI, test x$enable_gui = xyes) + + +AC_SUBST(RELP_CFLAGS) +AC_SUBST(RELP_LIBS) +# This provides a vehicle to integrate custom modules, that are not +# part of rsyslog, into the build process. It is named cust1, so that +# additional such modules can easily be added. +AC_ARG_ENABLE(cust1, + [AS_HELP_STRING([--enable-cust1],[Compiles stdout module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_cust1="yes" ;; + no) enable_cust1="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-cust1) ;; + esac], + [enable_cust1=no] +) +AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes) + + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -757,12 +910,7 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to search for omtemplate to find everything! -SHAVE_INIT - - AC_CONFIG_FILES([Makefile \ - shave \ - shave-libtool \ runtime/Makefile \ tools/Makefile \ doc/Makefile \ @@ -777,6 +925,8 @@ AC_CONFIG_FILES([Makefile \ plugins/omtemplate/Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ + plugins/omruleset/Makefile \ + plugins/omdbalerting/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ @@ -789,19 +939,24 @@ AC_CONFIG_FILES([Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ + plugins/omudpspoof/Makefile \ + plugins/cust1/Makefile \ + java/Makefile \ tests/Makefile]) AC_OUTPUT echo "****************************************************" echo "rsyslog will be compiled with the following settings:" echo -echo " Multithreading support enabled: $enable_pthreads" echo " Large file support enabled: $enable_largefile" echo " Networking support enabled: $enable_inet" echo " Regular expressions support enabled: $enable_regexp" echo " Zlib compression support enabled: $enable_zlib" echo " rsyslog runtime will be built: $enable_rsyslogrt" echo " rsyslogd will be built: $enable_rsyslogd" +echo " GUI components will be built: $enable_gui" +echo " custom module 1 will be built: $enable_cust1" +echo " Unlimited select() support enabled: $enable_unlimited_select" echo echo "---{ input plugins }---" echo " Klog functionality enabled: $enable_klog ($os_type)" @@ -813,6 +968,9 @@ echo "---{ output plugins }---" echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" +echo " omruleset module will be compiled: $enable_omruleset" +echo " omdbalerting module will be compiled: $enable_omdbalerting" +echo " omudpspoof module will be compiled: $enable_omudpspoof" echo " output template module will be compiled: $enable_omtemplate" echo echo "---{ database support }---" @@ -828,8 +986,10 @@ echo " RELP support enabled: $enable_relp" echo " SNMP support enabled: $enable_snmp" echo echo "---{ debugging support }---" +echo " Testbench enabled: $enable_testbench" echo " Debug mode enabled: $enable_debug" echo " Runtime Instrumentation enabled: $enable_rtinst" echo " Diagnostic tools enabled: $enable_diagtools" +echo " Enhanced memory checking enabled: $enable_memcheck" echo " Valgrind support settings enabled: $enable_valgrind" echo @@ -27,16 +27,12 @@ #ifndef DIRTY_H_INCLUDED #define DIRTY_H_INCLUDED 1 +rsRetVal multiSubmitMsg(multi_submit_t *pMultiSub); rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); -rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); -int parseRFCSyslogMsg(msg_t *pMsg, int flags); -int parseLegacySyslogMsg(msg_t *pMsg, int flags); +rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ - -/* TODO: the following 2 need to go in conf obj interface... */ -rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); -rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); +rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName); /* Intervals at which we flush out "message repeated" messages, * in seconds after previous message is logged. After each flush, @@ -44,31 +40,13 @@ rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pO * TODO: move this to action object! Only action.c and syslogd.c use it. */ extern int bActExecWhenPrevSusp; -extern int iActExecOnceInterval; extern int MarkInterval; extern int repeatinterval[2]; extern int bReduceRepeatMsgs; +extern qqueue_t *pMsgQueue; /* the main message queue */ #define MAXREPEAT ((int)((sizeof(repeatinterval) / sizeof(repeatinterval[0])) - 1)) #define REPEATTIME(f) ((f)->f_time + repeatinterval[(f)->f_repeatcount]) #define BACKOFF(f) { if (++(f)->f_repeatcount > MAXREPEAT) \ (f)->f_repeatcount = MAXREPEAT; \ } -extern int bDropTrailingLF; -extern uchar cCCEscapeChar; -extern int bEscapeCCOnRcv; -#ifdef USE_NETZIP -/* config param: minimum message size to try compression. The smaller - * the message, the less likely is any compression gain. We check for - * gain before we submit the message. But to do so we still need to - * do the (costly) compress() call. The following setting sets a size - * for which no call to compress() is done at all. This may result in - * a few more bytes being transmited but better overall performance. - * Note: I have not yet checked the minimum UDP packet size. It might be - * that we do not save anything by compressing very small messages, because - * UDP might need to pad ;) - * rgerhards, 2006-11-30 - */ -#define MIN_SIZE_FOR_COMPRESS 60 -#endif - #endif /* #ifndef DIRTY_H_INCLUDED */ diff --git a/doc/Makefile.am b/doc/Makefile.am index 0703b8fc..42a0e83c 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -13,13 +13,13 @@ html_files = \ ipv6.html \ log_rotation_fix_size.html \ manual.html \ - man_rsyslogd.html \ modules.html \ property_replacer.html \ rsyslog_ng_comparison.html \ rsyslog_conf.html \ rsyslog-example.conf \ rsyslog_mysql.html \ + rsyslog_pgsql.html \ rsyslog_packages.html \ rsyslog_high_database_rate.html \ rsyslog_php_syslog_ng.html \ @@ -31,6 +31,8 @@ html_files = \ version_naming.html \ contributors.html \ dev_queue.html \ + omstdout.html \ + omudpspoof.html \ omsnmp.html \ ommysql.html \ omoracle.html \ @@ -64,6 +66,7 @@ html_files = \ rsconf1_actionresumeinterval.html \ rsconf1_allowedsender.html \ rsconf1_controlcharacterescapeprefix.html \ + rsconf1_escape8bitcharsonreceive.html \ rsconf1_debugprintcfsyslinehandlerlist.html \ rsconf1_debugprintmodulelist.html \ rsconf1_debugprinttemplatelist.html \ @@ -89,8 +92,12 @@ html_files = \ rsconf1_moddir.html \ rsconf1_repeatedmsgreduction.html \ rsconf1_resetconfigvariables.html \ + rsconf1_rulesetcreatemainqueue.html \ rsconf1_umask.html \ + rsconf1_rulesetparser.html \ v3compatibility.html \ + v4compatibility.html \ + v5compatibility.html \ im3195.html \ netstream.html \ ns_gtls.html \ @@ -111,11 +118,13 @@ html_files = \ rsyslog_conf_templates.html \ rsyslog_conf_nomatch.html \ queues_analogy.html \ + multi_ruleset.html \ src/classes.dia grfx_files = \ rsyslog_confgraph_complex.png\ rsyslog_confgraph_std.png \ + module_workflow.png \ direct_queue0.png \ direct_queue1.png \ direct_queue2.png \ @@ -126,6 +135,8 @@ grfx_files = \ dataflow.png \ queue_analogy_tv.png \ gssapi.png \ + rfc5424layers.png \ + src/rfc5424layers.dia \ rsyslog-vers.png EXTRA_DIST = $(html_files) $(grfx_files) diff --git a/doc/action-call.dot b/doc/action-call.dot new file mode 100644 index 00000000..86c6834d --- /dev/null +++ b/doc/action-call.dot @@ -0,0 +1,33 @@ +// This file is part of rsyslog. +// +// rsyslog action call state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot action-call.dot -Tpng >action-call.png + +digraph G { + label="\n\nrsyslog message states during action processing\nhttp://www.rsyslog.com"; + //fontsize=20; + + ok [label="ready for processing" color="green"]; + mpf [label="message permanent failure" color="red"]; + tf [label="temporary failure"] + cPen [label="commit pending"]; + com [label="committed" color="red"]; + + tf -> tf [label="retry fails, i < n"]; + tf -> mpf [label="retry fails, i = n"]; + tf -> ok [label="retry succeeds"]; + ok -> com [label="doAction RS_RET_OK"]; + ok -> cPen [label="doAction COMMIT_PENDING"]; + ok -> tf [label="doAction RS_RET_SUSPENDED"]; + ok -> mpf [label="doAction RS_RET_DISABLED"]; + cPen -> com [label="endTransaction RS_RET_OK"]; + cPen -> tf [label="endTransaction _SUSPENDED"]; + + //{rank=same; tf cPen} + {rank=same; com mpf} +} diff --git a/doc/action_state.dot b/doc/action_state.dot new file mode 100644 index 00000000..2f36d8da --- /dev/null +++ b/doc/action_state.dot @@ -0,0 +1,34 @@ +// This file is part of rsyslog. +// +// rsyslog message state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog action transaction states\nhttp://www.rsyslog.com"; + //fontsize=20; + + rdy [label="ready" group="main"]; + itx [label="in Tx" group="main"]; + comm [label="commit"] + rtry [label="retry"] + susp [label="suspended"] + + rdy -> itx [label="transaction begins"] + rdy -> rtry [label="begin tx\nerror"] + itx -> itx [label="success"] + itx -> comm [label="commit\n(caller or auto)"] + itx -> rtry [label="error"] + comm -> rdy [label="success"] + comm -> rtry [label="error"] + rtry -> rdy [label="recovered"] + rtry -> susp [label="could not\nrecover"] + susp -> rtry [label="timeout expired"] + + {rank=same; comm rtry} +} diff --git a/doc/batch_state.dot b/doc/batch_state.dot new file mode 100644 index 00000000..0dd48b47 --- /dev/null +++ b/doc/batch_state.dot @@ -0,0 +1,28 @@ +// This file is part of rsyslog. +// +// rsyslog batch state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog batch states\nhttp://www.rsyslog.com"; + rankdir=LR + + rdy [label="ready"]; + bad [label="message-caused\nfailure"]; + sub [label="submitted"] + disc [label="discarded" color="red"] + + rdy -> sub [label="submitted to action"] + rdy -> bad [label="permanent fail"] + rdy -> disc [label="action requests discarding"] + sub -> rdy [label="next action or\naction-caused failure"] + bad -> rdy [label="next action"] + + //{rank=same; comm rtry } +} diff --git a/doc/build_from_repo.html b/doc/build_from_repo.html index 8d3b20fe..a06863e1 100644 --- a/doc/build_from_repo.html +++ b/doc/build_from_repo.html @@ -43,12 +43,37 @@ you downloaded an official distribution tarball (see the <a href="install.html">rsyslog install guide</a>, starting at step 2, for further details about that). +<h2>Special Compile-Time Options</h2> +<p>On some platforms, compile-time issues occur, like the one shown below: +<p><pre><code> +make[2]: Entering directory `/home/az/RSyslog/rsyslog-5.5.0/tools' + CCLD rsyslogd +rsyslogd-omfile.o: In function `getClockFileAccess': +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +</code></pre> +<p>Note that the exact error messages can be different. These type of errors stem down to +atomic instruction support in GCC, which is somewhat depending on the machine architecture it +compiles code for. Very old machines (like the original i386) do not even at all provide support +for these instructions. +<p>The availability of atomic instructions is vital for rsyslog - it can not be built without them. +Consequently, there is a configure check included for them. But under some circumstances, +GCC seems to report they are available, but does not provide implementations for +all of them (at least this is my observation...). The simple cure is to make sure that +GCC generates code for a modern-enough architecture. This, for example, can be done as +follows: +<p><pre><code> +./configure CFLAGS="-march=i586 -mcpu=i686" --enable-imfile ... (whatever you need) +</code></pre> +<p>These settings should resolve the issue . + <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 1.2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/debug.html b/doc/debug.html index de77f04a..6aeb7975 100644 --- a/doc/debug.html +++ b/doc/debug.html @@ -1,41 +1,145 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html><head> -<meta http-equiv="Content-Language" content="en"><title>Debug Support</title></head> +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>Rsyslog Debug Support</title></head> <body> -<h1>Debug Support</h1> +<h1>Rsyslog Debug Support</h1> <p> -Rsyslog provides a number of debug aids. Some of them are activated by +Rsyslog provides a number of debug aides. Some of them are activated by adding the --enable-rtinst ./configure option ("rtinst" means runtime instrumentation). Turning debugging on obviously costs some performance (in some cases considerable). </p> <p>This is document is just being created and thus terse.</p> -<p style="font-weight: bold;">Signals supported</p> -<p>SIGUSR1 - turns debug messages on and off (expect this signal -to go away over time)</p> -<p>SIGUSR2 - outputs debug information (including active threads +<h2>Signals supported</h2> +<p><b>SIGUSR1</b> - turns debug messages on and off. Note that for this +signal to work, rsyslogd must be running with debugging enabled, either +via the -d command line switch or the environment options specified below. +It is <b>not</b> required that rsyslog was compiled with debugging enabled +(but depending on the settings this may lead to better debug info). +<p><b>SIGUSR2</b> - outputs debug information (including active threads and a call stack) for the state when SIGUSR2 was received. This is a -one-time output. Can be sent as often as the user likes.</p> -<p style="font-weight: bold;">Environment Variables</p> -<p>There are two environment variables that set several debug settings. The "RSYSLOG_DEBUGLOG" (sample: RSYSLOG_DEBUGLOG="/path/to/debuglog/") +one-time output. Can be sent as often as the user likes. +<p><b>Note:</b> this signal <b>may go away</b> in later releases and may +be replaced by something else.</p> +<h2>Environment Variables</h2> +<p>There are two environment variables that set several debug settings: +<ul> +<li>The "RSYSLOG_DEBUGLOG" (sample: RSYSLOG_DEBUGLOG="/path/to/debuglog/") writes (allmost) all debug message to the specified log file in addition to stdout. Some system messages (e.g. segfault or abort message) are not written to the -file as we can not capture them. Runtime debug support is controlled by -"RSYSLOG_DEBUG". It contains an option string with the following -options possible (all are case insensitive):</p><ul><li><span style="font-weight: bold;">LogFuncFlow</span> - print out the logical flow of functions (entering and exiting them)</li><li><span style="font-weight: bold;">FileTrace</span> - specifies which files to trace LogFuncFlow. If <span style="font-weight: bold;">not</span> +file as we can not capture them. +<li>Runtime debug support is controlled by "RSYSLOG_DEBUG". +<p>The "RSYSLOG_DEBUG" environment variable contains an option string with the following +options possible (all are case insensitive):</p> +<ul> +<li><b>LogFuncFlow</b> - print out the logical flow of functions (entering and exiting them)</li> +<li><b>FileTrace</b> - specifies which files to trace LogFuncFlow. If <b>not</b> set (the default), a LogFuncFlow trace is provided for all files. Set to limit it to the files specified. FileTrace may be specified multiple times, one file each (e.g. export RSYSLOG_DEBUG="LogFuncFlow -FileTrace=vm.c FileTrace=expr.c"</li><li><span style="font-weight: bold;">PrintFuncDB</span> - print the content of the debug function database whenever debug information is printed (e.g. abort case)!</li><li><span style="font-weight: bold;">PrintAllDebugInfoOnExit</span> - print all debug information immediately before rsyslogd exits (<span style="font-weight: bold; font-style: italic;">currently not implemented!</span>)</li><li><span style="font-weight: bold;">PrintMutexAction</span> - print mutex action as it happens. Useful for finding deadlocks and such.</li><li><span style="font-weight: bold;">NoLogTimeStamp</span> - do not prefix log lines with a timestamp (default is to do that).</li><li><span style="font-weight: bold;">NoStdOut</span> - do not emit debug messages to stdout. If RSYSLOG_DEBUGLOG is not set, this means no messages will be displayed at all.</li><li><span style="font-weight: bold;">help</span> - display a very short list of commands - hopefully a life saver if you can't access the documentation...</li></ul> +FileTrace=vm.c FileTrace=expr.c"</li> +<li><b>PrintFuncDB</b> - print the content of the debug function database whenever debug information is printed (e.g. abort case)!</li> +<li><b>PrintAllDebugInfoOnExit</b> - print all debug information immediately before rsyslogd exits (<span style="font-weight: bold; font-style: italic;">currently not implemented!</span>)</li> +<li><b>PrintMutexAction</b> - print mutex action as it happens. Useful for finding deadlocks and such.</li> +<li><b>NoLogTimeStamp</b> - do not prefix log lines with a timestamp (default is to do that).</li> +<li><b>NoStdOut</b> - do not emit debug messages to stdout. If RSYSLOG_DEBUGLOG is not set, this means no messages will be displayed at all.</li> +<li><b>Debug</b> - if present, turns on the debug system and enables debug output +<li><b>DebugOnDemand</b> - if present, turns on the debug system but does not enable +debug output itself. You need to send SIGUSR1 to turn it on when desired. +<li><b>help</b> - display a very short list of commands - hopefully a life saver if you can't access the documentation...</li> +</ul> +</ul> +<h3>Why Environment Variables?</h3> +<p>You may ask why we use environment variables for debug-system parameters and not +the usual rsyslog.conf configuration commands. After all, environment variables force one +to change distro-specific configuration files, whereas regular configuration directives +would fit nicely into the one central rsyslog.conf. +<p>The problem here is that many settings of the debug system must be initialized +before the full rsyslog engine starts up. At that point, there is no such thing like +rsyslog.conf or the objects needed to process it present in an running instance. +And even if we would enable to change settings some time later, that would mean that +we can not correctly monitor (and debug) the initial startup phase of rsyslogd. What +makes matters worse is that during this startup phase (and never again later!) some +of the base debug structure needs to be created, at least if the build is +configured for that (many of these things only happen in --enable-rtinst mode). So +if we do not initialize the debug system <b>before</b> actually startig up the +rsyslog core, we get a number of data structures wrong. +<p>For these reasons, we utilize environment variables to initialize and configure +the debugging system. We understand this may be somewhat painful, but now you know +there are at least some good reasons for doing so. +<h2>Getting debug information from a running Instance</h2> +<p>It is possible to obtain debugging information from a running instance, but this requires +some setup. We assume that the instance runs in the background, so debug output to +stdout is not desired. As such, all debug information needs to go into a log file. +<p>To create this setup, you need to <ul> +<li>point the RSYSLOG_DEBUGLOG environment variable to a file that is accessible +during the while runtime (we strongly suggest a file in the local file system!) +<li>set RSYSLOG_DEBUG at least to "DebugOnDeman NoStdOut" +<li>make sure these environment variables are set in the correct (distro-specifc) +startup script if you do not run rsyslogd interactively </ul> +<p>These settings enable the capability to react to SIGUSR1. The signal will toggle +debug status when received. So send it one to turn debug loggin on, and send it again +to turn debug logging off again. The third time it will be turned on again ... and so on. +<p>On a typical system, you can signal rsyslogd as follows: +<pre> +kill -USR1 `cat /var/run/rsyslogd.pid` +</pre> +Important: there are backticks around the "cat"-command. If you use the regular +quote it won't work. The debug log will show whether debug logging has been turned +on or off. There is no other indication of the status. +<p>Note: running with DebugOnDemand by itself does in practice not have any performance +toll. However, switching debug logging on has a severe performance toll. Also, debug +logging synchronizes much of the code, removing a lot of concurrency and thus +potential race conditions. As such, the very same running instance may behave +very differently with debug logging turned on vs. off. The on-demand debug log +functionality is considered to be very valuable to analyze hard-to-find bugs that +only manifest after a long runtime. Turning debug logging on a failing instance +may reveal the cause of the failure. However, depending on the failure, debug logging +may not even be successfully be turned on. Also note that with this rsyslog version we cannot +obtain any debug information on events that happened <i>before</i> debug logging was +turned on. +<p>If an instance hangs, it is possible to obtain some useful information about the current +threads and their calling stack by sending SIGUSR2. However, the usefulness of that +information is very much depending on rsyslog compile-time settings, must importantly +the --enable-rtinst configure flag. Note that activating this option causes additional overhead +and slows down rsyslgod considerable. So if you do that, you need to check if it is +capable to handle the workload. Also, threading behavior is modified by the +runtime instrumentation. +<p>Sending SIGUSR2 writes new process state information to the log file each time +it is sent. So it may be useful to do that from time to time. It probably is most +useful if the process seems to hang, in which case it may (may!) be able to output +some diagnostic information on the current processing state. In that case, turning +on the mutex debugging options (see above) is probably useful. +<h2>Interpreting the Logs</h2> +<p>Debug logs are primarily meant for rsyslog developers. But they may still provide valuable +information to users. Just be warned that logs sometimes contains informaton the looks like +an error, but actually is none. We put a lot of extra information into the logs, and there +are some cases where it is OK for an error to happen, we just wanted to record it inside +the log. The code handles many cases automatically. So, in short, the log may not make sense to +you, but it (hopefully) makes sense to a developer. Note that we developers often need +many lines of the log file, it is relatively rare that a problem can be diagnosed by +looking at just a couple of (hundered) log records. +<h2>Security Risks</h2> +<p>The debug log will reveal potentially sensible information, including user accounts and +passwords, to anyone able to read the log file. As such, it is recommended to properly +guard access to the log file. Also, an instance running with debug log enabled runs much +slower than one without. An attacker may use this to place carry out a denial-of-service +attack or try to hide some information from the log file. As such, it is suggested to +enable DebugOnDemand mode only for a reason. Note that when no debug mode is enabled, +SIGUSR1 and SIGUSR2 are completely ignored. +<p>When running in any of the debug modes (including on demand mode), an interactive +instance of rsyslogd can be aborted by pressing ctl-c. +<p> <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the -<a href="http://www.rsyslog.com/">rsyslog</a> -project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html>
\ No newline at end of file +</body> +</html> diff --git a/doc/design.tex b/doc/design.tex new file mode 100644 index 00000000..a3ec8f45 --- /dev/null +++ b/doc/design.tex @@ -0,0 +1,887 @@ +\documentclass[a4paper,10pt]{article} +\usepackage{amsmath} +\usepackage{amsfonts} +\usepackage{amssymb} +\usepackage{graphicx} +\usepackage{listings} +\usepackage{algorithm,algorithmic} +\usepackage{float} + +\pagestyle{headings} + +\newcommand{\IN}{\mathbb{N}} +\newcommand{\MM}{\mathcal{M}} +\newcommand{\QQ}{\mathcal{Q}} +\newcommand{\AAA}{\mathcal{A}} +\title{Rsyslog Design and Internals} +\author{Rainer Gerhards\\ +rgerhards@adiscon.com} + +\begin{document} + +\maketitle + +\begin{abstract} +This paper describes rsyslog design and internals. It is created to facilitate a discussion about the implementation of "batched queue processing". As such, it does not describe the full design of rsyslog but rather those elements that are relevant to queues. However, the document may be expanded in the future. This is work in progress and should be considered with care. It is NOT updated during all phases of development. +\end{abstract} + +\tableofcontents + +\section{Preliminaries} +\subsection{On the Use of English} +\begin{quotation} +\begin{flushright} +I ventured to write this book in English because ... \\ +it will be more easily read in poor English, \\ +than in good German by 90\% of my intended readers. \\ +--- HANS J. STETTER, Analysis of Discretization Methods for \\ +Ordinary Differential Equations (1973) +\end{flushright} +\end{quotation} + +There is not much I could add to Mr. Stetter's thought, except, maybe, that the number to quote probably tends more to 99\% in this case than to the 90\% Mr. Stetter notes. So please pardon those errors in language use that I have not yet been able to fix or even see. Suggestions for corrections and improvements are always welcome. +\subsection{Notational Conventions} +In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathcal{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in caligraphic letters ($\mathcal{O}$). Often, objects $O_i, i \in \IN, i \le |\mathcal{O}|$ partition $\mathcal{O}$, but this is not necessarily the case. + +\subsection{Definitions} +\subsubsection{Sudden Fatal Failure} +As sudden fatal failure is one that occurs at some instant and causes Complete loss of processing capabilities. The two major cases are a sudden power loss or a ``kill -9'' of the process. There are more exotic cases, too, like disasters. + +One may argue that it is possible to protect against many sudden fatal failure cases. For example, using an uninterruptable power supply (UPS) will prevent a sudden power loss. While this is true in most cases, it does not hold if looked very closely: in the case of the UPS, for example, a failure in the UPS itself may cause a sudden power loss, which can not be mitigated. Well, actually there can be several layers of mitigation, but always one more potential failure scenario remains. So it is not possible to totally solve the issue. + +The concept of ``sudden fatal failure'' now covers all these rest risk that result in termiantion of rsyslogd without the ability execute any code before this happens. This is a very important concept in regard to audit-gradeness. + +\subsubsection{Audit Grade} +In the context of this document, ``audit grade'' means that a subsystem never loses a message that it has taken responsibility for, not even in cases of sudden fatal failures. The only limit in this restriction is that a subsystem does not guarantee message survival if the subsytem at large is being destroyed (e.g. during a disaster) or some of its components are not of audit-grade. This draws a fine limitation on the audit-grade of a subsystem. + +For example, the rsyslog queue subsystem receives messages and acknowledges them to the submitter (e.g. an input), when they have been enqueued in the storage system. If the queue system is configured to provide audit-grade operation\footnote{Audit-grade queue operation is considerably slower than regular operations, as such this mode is not enabled by default. Most installations will never need a completely audit-grade queue}, the queue relies on the storage subsystem to work properly. If, for example, a disk read error occurs, the message may no longer be readable from the disk and as such is lost. The root cause here is that the disk subsystem was not of audit grade, because it otherwise would not have lost the message. So in this case the queue code is of audit grade, but the one of its components, the disk subsytem, was not. So the overall system is not of audit grade. + +To simplify talking about the audit-gradness of several subsytems, we assume that all of their subsystems are also of audit grade. In an actual deployment, however, this means the the system designer must carefully select audit-grade subsystems. Overlooking a single non-audit-grade component will make the whole system of not audit grade quality. + +Please note that it can be rather tricky to ensure a complete system is of audit grade. A border case is main memory integrity. Even with error-correcting memory, there may situations arise where a memory error occurs (probably due to a very unlikely series of well-hitting cosmic rays) that is unrecoverable. At this point, system integrity is at risk. The only real solution is to immediately shut down the system and restart it (without giving any process a chance to execute). Note, however, that in an extreme view, an operating system routine that does so can also be considered dangerous, as memory in use by this routine might be affected by the malfunction. We could extend this scenario and further complicate it, but that goes beyond the scope of this paper. The example was primarily meant to show how subtle audit-grade reliability is. + +In rsyslog, we currently use a slightly \marginpar{duplication\\permitted}relaxed consistency condition for message integrity inside an audit-grade subsystem. While we do not accept message loss, we permit slight message \emph{duplication}, but only in exceptional cases. This is permitted because, with proper message generation, the dulication problem can be easily fixed at the end-to-end layer. For example, the original sender can include a UUID, which can be used to sort out duplicates at the final destination. Insisting on not allowing duplication complicates matters and is often impossible with today's logging protocols. So, for the time being, we aim at this relaxed criteria, which is hard enough to achive. After we have achieved that goal, we may further try to solve the duplicaton problem. Some hooks already exist. But we do not guarantee such an effort will be made any time soon. + +\section{Overall Design} +From a high-level prespective, rsyslogd is ``just'' a high-performance message router. It accepts messages from various sources, applies user-configured filters to them, and routes potentially transformed messages to destinations based on these filters. +\section{Objects} +\subsection{Plugins} +Plugins provide code potentially written by a third party to extend rsyslog. + +Conceptually, a plugin is a tuple of callable functions $(\phi_1, \phi_2, \ldots)$ which implement an interface. There are three different types of plugins: input, output and library. The plugin type denotes the primary interface implemented by the plugin. Additional interfaces may be implemented\footnote{This is not yet done in plugins, but is possible and assumed to be done at a later point in time}. + +In the context of this paper, the output plugin interface is most important. It implements three entry points: + +\paragraph{doAction()} +is used to submit messages to the output plugin. The entry point may or may not commit the messages to their ultimate destination. + +\paragraph{beginTransaction()} +is used to inform the plugin that a new transaction begins. It must prepare for processing. + +\paragraph{endTransaction()} +is indicated that the upper layer \emph{needs} to close the transaction. If there is any uncommited data left, it must be commited or rolled back. + +Every instance of an output plugin is guaranteed \emph{not} to be called concurrently by multiple threads. Further, no context switch will happen between calls to $doAction()$ and $endTransaction()$. + +\subsection{State Sets} +Several object have associated state based on a specific state set. These state sets are described together with the objects. + +As a general rule, individual state is associated with all instances $o$ of a class of objects. This state is called the object's \marginpar{state component} \emph{state component} $s$. If we want to obtain an object's state, we write $S(o)$. Please note that $S(o)$ is only defined for those objects that have a state component. + +\subsection{Messages} +A message $m$ represents a a single syslog message inside the system. It is a tuple of attributes. Some of these attributes directly orginate from the message content, some others are meta-information taken from the context. For example, there is an meta-attribute ``time of reception'' which conveys when the message was received by rsyslog's input subsystem. We do not list attributes here, as there are many and it is not of importance which exactly they are. + +The set $\MM$ is composed of all messages that exist at a given time inside rsyslog. + +\subsection{Queue} +A queue +$$Q = (C, \Phi, M)$$ +is a triplet of a set of configuration parameters $C$, a set of callbacks $\Phi$ and a set of messages $M \subseteq \MM$. + +If we need to obtain the set of message from a queue, we write $M(Q)$. The elements of the set of configuration parameters are written as $C_{param}$ where $param$ is an abbreviation of the parameter's meaning. To obtain a specific parameter from a queue, we write $C_{param}(Q)$. The most important elements of $C$ are: + +\paragraph{$C_{type}$} which denotes the queue implementation type. Most importantly, this selects from a set of queue drivers (for example disk-only or in-memory driver), which affects the basic operation of the queue instance. + +\paragraph{$C_{mMsg}$} which denotes the upper bound on the cardinality of $M$. + +\paragraph{$C_{mBatch}$} which denotes the upper bound of the cardinality of message batches created for this queue. + +Be $\QQ = \{Q_m, Q_1, Q_2, \ldots, Q_{|\AAA|}\}$ the set of all queues that exist inside rsyslog after the configuration file has been processed, with $|\QQ| = |\AAA| + 1$. + +Then +$$M_0 = \MM \setminus \bigcup_{i=1}^{|\QQ|} Q_i(M)$$ +\marginpar{at-risk-set}is the set of non-queued messages. The messages have either never been enqueued or have been dequeued but not finally been processed. This set represents the messages that may potentially be lost during an unclean shutdown of rsyslogd. This is why I call this set the ``\emph{at-risk-set}''. + + +\subsection{Batches} +A batch represents multiple processable messages. It is a unit of processing inside rsyslog's output system. Batches are used to dequeue a number of messages from a queue and then submit them to the lower action layer. Batches are natural \emph{transaction boundaries}, in the sense that multiple output transactions may be done on the messages inside a batch, but each transaction must end at the end of the batch. A batch is always associated to a specific queue $Q$. + +A batch +$$B = (b_1, b_2, \ldots, b_n )$$ +is a $n$-tuple of \marginpar{processable\\message}processable messages +$$b = (m, s)$$ +which are an ordered pair of a message $m$ and an associated processing state $s$. To denote the $n$-th message inside the batch, we write $m(b_n)$, to denote the status component of the $n$-th message, we write $S(b_n)$. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{batch_state.jpeg} +\end{center} +\caption{batch message processing states} +\label{fig_batchmsg_states} +\end{figure} + +The state set for the processing states is defined as follows: +$$ +S_B = \{ rdy, bad, sub, disc \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready for processing\\ + bad & this message triggered an unrecoverable failure in action\\ + & processing and must not be resubmitted to this action\\ + sub & message submitted for processsing, result yet unknown \\ + disc & action sucessfully processed, but must not be submitted \\ + & to any further action in action unit \\\hline +\end{tabular} +\end{center} +The associated state diagram is shown in figure \ref{fig_batchmsg_states} on page \pageref{fig_batchmsg_states}. + +Batch sizes vary. The actual cardinality is a function of the cardinality of $M(Q)$ at the time of batch creation and the queue configuration: + +$$1 \leq |B| \leq \max(C_{mBatch}(Q), |M(Q)|)$$ + +\subsection{Action Unit} +An action unit +$$u = (f, a_1, \ldots, a_n), a_i \in \AAA \text{ for } i \in \IN, i \le n$$ +is a tuple consisting of a filter function $f$ and $n \in \IN$ actions. \emph{Does rsyslog still support nonsense action units with $n=0$? - check!} + +\subsection{Action} +An action +$$a = (a_C, a_\psi)$$ +is an ordered pair of a tuple of configuration attributes $a_C$, and a tuple of processing functions $a_\psi$. Be the set $\AAA$ composed of all actions that exist in rsyslog after the configuration file has been processed. + + +\section{Processing} +\subsection{Object States} +Various objects keep state. Some of these objects, like messages, batches and actions seem to share state. However, thinking about shared state leads to very complex setup. As such, state is modelled for each object $o$ individually. Instead, the state function $S_O(o)$ can be used to obtain an obtain an individual objects state. That state can be used to modify the state diagrams of the other objects with which relationships exist. + +\subsubsection{Actions} +Actions are provided by output plugins. An action enables the engine to write messages to some destination. It is important to note that ``destination'' is a very broad abstraction. A destination may be a file inside a local or remote file system, a database table or a remote syslog server in another network. + +Actions are transactional in the following sense: more than one message can be submitted to an action. The action does not necessarily process the submitted messages unless the caller ends the transaction. However, the action itself may also end the transaction and notify the caller. This is \emph{not} considered an error condition and \emph{must} be handled gracefully by the caller. If a transaction aborts, the caller \emph{must} assume that none of the elements submitted since the begin of transaction have been processed. The action will try to backout anything that was already processed at the time the transaction failed. However, not all outputs work on actually transactional destination. As such, an action is permitted not to backout incomplete interim results. As such, after a transaction abort, some message duplication may occur. We call this the \emph{relaxed integrity condition} for actions. + +An output transaction is started by calling \emph{beginTransaction()} either explicitely or implicitely by a call to \emph{doAction()} without calling \emph{beginTransaction()} before. Then, one or more calls to \emph{doAction()} follow. When the caller intends to finish the transaction, it calls \emph{endTransaction()}. However, the transaction may also be terminated from the action itself in response to a \emph{doAction()} call. + +Mathematically, an action transaction builds a totally ordered set of uncommitted messages $M_u$. The order relation is defined over the sequence in which messages are being provided to \emph{doAction()}. At any time a commit is attempted, the full set $M_u$ is committed and may either succeeed completely or not at all (in the sense of the relaxed integrity condition described above). + +A commit is attempted when +\begin{enumerate} +\item the caller decides to call \emph{endTransaction()} +\item or earlier if the action decides it needs to commit now (e.g. because of buffers filling up). +\end{enumerate} + +In the seconds case, the action may decide to commit all message but the current one or all (this is depending on action logic). So if the action decideds to commit a transaction before the caller calls \emph{endTransaction()}, a set of commited messages $M_c$ is build and $M_u$ is modified. Be $n$ the $n$-th iterated \emph{doAction()} call and $m_n$ the current message of this call, then the sets are build as follows: + +\begin{algorithm} +%\caption{} +\begin{algorithmic} +\IF{action commits $m_n$} + \STATE $M_c = M_u \cup m_n$ + \STATE $M_u = \emptyset$ +\ELSE + \STATE $M_c = M_u$ + \STATE $M_u = \{ m_n\}$ +\ENDIF +\end{algorithmic} +\end{algorithm} + +In other words, if anything is committed early, it is always the full set $M_u$, with or without the current message. The caller needs to know which messages are already commited. As \emph{doAction()} finishes one transaction and starts a new one in a single call, we can not use action state the let the caller know this happened. So we use our above finding and just convey back if the transacton is still continuing or the current message or all others before it were committed. The caller must then act accordingly. Please note that when an error happens, the whole transaction must still be considered failed. As such, ``partial commit'' states need not to be mixed with failure states. + +Please note that the above method leaves a small potential issue unaddressed: if the action does an early commit of $M_u \setminus m_n$, an error happens when adding $m_n$ to the new $M_u$ (like running out of resources), the action would need to convey both the successful transaction as well as the failure state. This is not possible with the current interface. We could use callbacks to provide such notification, but this complicates the code. So, if that situaton arises, the action must temporarily buffer the error condition and convey it as part of either the next \emph{doAction()} call or during \emph{endTransation()} processing. This can be done, for example, by advancing its internal state accordingly. + +The state set for a actions is defined as follows: +$$ +S_A = \{ rdy, itx, comm, rtry, susp, died \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready, waiting for transaction begin\\ + itx & in transaction, accept more data \\ + comm & transaction finished \\ + rtry & action failed but may be able to recover \\ + susp & action currently defunctional until timeout expires \\ + died & unrecoverable error condition occured, no longer usable \\\hline +\end{tabular} +\end{center} + +In the associated state diagram in figure \ref{fig_action_states}, we do not include the \emph{died} state, because it is entered whenever a totally unrecoverable error state may occur. This is a very exceptional incident (which most output plugins do not even support), so we have kept the diagram simple. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.5]{action_state.jpeg} +\end{center} +\caption{Action State Diagram} +\label{fig_action_states} +\end{figure} + +\emph{Note well} that the state diagram describes the action state. It does \emph{not} describe the transaction state. While action- and transaction state are closely related to each other, they are different entities. + +The return code of \emph{doAction()} and \emph{endTransaction()} is used to convey the transaction state. As such, it is a function of the actions's current state after processing the request. The mapping is as shown below: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Return Code (RS\_RET\_\ldots)\\\hline + rdy & OK \\ + itx & COMMITTED (if there was an auto-commit without $m_n$)\\ + & DEFER\_COMMIT (if there was no auto-commit)\\ + comm & internal state, not to be exposed to upper layer \\ + rtry & SUSPENDED \emph{(new code needed)} \\ + susp & SUSPENDED \\ + died & DISABLED \\\hline +\end{tabular} +\end{center} + +For the rest of this document, let's assume there is a function \emph{getReturnCode()} that implements this mapping. + +It is important to think about how retries are handled. There is a user-configured per-action upper number of retries $C_r$ and retry interval $C_i$. In \emph{rsyslog v3}, there is no concept of output transactions. As such, only single messages are processed. When a temporary action failure occurs, the action is re-tried $C_r$ times, where the action processing thread is waiting in a \emph{sleep()} $C_i$ operating system API call\footnote{a suitable API is used, not \emph{sleep()} itself}. If the action succeeds during the retry processing, everything continues as usual. If it does not succeed, two things happen: +\begin{itemize} +\item the message is flagged as ``action permanent failure'' (what may trigger backup processing) +\item the action is actually suspended for $C_i$ seconds +\end{itemize} +If then a new message is sent to the action, and $C_i$ seconds have not yet elapsed, the action is flagged as having failed without being re-tried again\footnote{During the analysis for this paper, it was seen that actually $C_r$ retries are attempted in v3, but each of them will never actually re-try the action. This is a software bug, which does not cause any harm and thus will not be fixed in v3. The new implementation in v4 will obviously not inherit this problem}. This is done in an effort to reduce resource utilization and prevent the system from slowing down e.g. by too-many retries to a remote server that went offline. + +With transactional output mode in \emph{rsyslog v4}, the logic above can no longer work. First of all, retrying single actions does not help, because all of the current transaction needs to be resubmitted. As such, the upper layers need to be notified of failure. Then, they need to resubmit the batch. In that design, the lower layer needs to return immediately after detecting the failure. Recovery handling is now to be done when the next transaction is started. However, we must make sure that we do not do excessive retries. So retry processing is only to be carried out if it was not tried less than $C_i$ seconds ago. + +The required functionality can be implemeted by a \emph{prepareAction} function that readies the action for processing if there is need to do so. That function is then called in all entry points before anything else is done. Then, actual processing is carried out and the resulting action state be used to generate the return code for the upper-layer caller. Find below a rough pseudocode to do so: + +\lstset{language=python} +\begin{lstlisting} +def prepareAction(): + if state == rtry: + try recovery (adjust state accordingly) + if state == rdy: + beginTransaction() [output plugin] + +def processMessage(message): + prepareAction() + if state == itx + doAction(message) [output plugin] + return getReturnCode() + +def doEndTransaction(): + prepareAction() + if state == itx + endTransaction(); [output plugin] + return getReturnCode() +\end{lstlisting} + +\subsection{Output Subsystem Layers} +The rsyslog engine is organized in layers, where each layer is represented by the dominating object: + +\begin{figure} +\includegraphics[scale=0.75]{rsyslog_output_layers.jpeg} +\label{rsyslog output layers} +\end{figure} + +If looking at the data flow, a queue dequeues batches of messages, which are than run through a generic action system and put into output plugins. Note that on the batch layer, only batches are supported as units of work, whereas the action layer is message-oriented but supports transactions of multiple messages. This is done by indicating when a transaction necessarily needs to end (that point being the end of batch from the batch layer). + +The plugins can be written by third parties and are roughly comparable to minidrivers. The generic action system provides all complexity of action processing wheras the output plugin provides a limited set of callbacks that enable the generic framework to talk to the actual destination system. As such, writing outputs is a very simple task. However, rsyslog does not limit the creation of very complex outputs, which may be able to offer superior performance for some destinations. + +\subsection{Output Failure} +\subsubsection{Cases} +When an output action is called, it may encounter a failure condition. In general, there are two different cases: +\begin{enumerate} +\item action caused failures +\item message-content caused failures +\end{enumerate}. + +Failures rooted in the action are things like broken network connections, file systems run out of space or database servers that are down. Most importantly, the failure is not related to message content. As such, it is appropriate to retry the action with the same message until it finally succeeds (assuming that someone restores the system in question to proper operation). We can not expect that the problem is cleared just by discarding the current message and re-trying with the next one. + +In my view, action caused failures are the far majority of all failures. For rsyslog versions 3 and below, all rsyslog-provided plugins consider failures to be action-caused and thus potentially recoverable by simple retry. With the only exception being fatal error conditions that render the whole action unusable. + +David Lang pointed out, that there may also exist error conditions that are not caused by the action (or the subsystem it talks to) itself, but rather by message data. He provided the following samples where message content can cause permanent issues with action execution: + +\begin{itemize} +\item unicode text causing grief +\item dynafile hits a read-only file +\item basicly data-driven things that trigger bugs in the message delivery +mechanism in some form. +\end{itemize} + +As David Lang said ``In an ideal world these would never happen, but for most output types I can think of some form of corrupt input that could cause that message to fail.''. +So this class of failure conditions actually exists. No matter how often the action retry mechanism is called, it will never succeeds (one may argue that the read-only dynafile is fixable, but we could replace that sample with an invalidly generated filename). The proper cure for these actions is to find the offending one and discard it. + +In conclusion, actions need to return different error states for these two different types of failures. Traditionally, RS\_RET\_SUSPENDED is returned when an action specific failure is hit. Most existing plugins also do this if a message-related failure occured, simply because they did not yet know that this situation exists. However, plugins also return different error codes, and at least these can be treated to mean message-permanent failures. To support this, a change to plugins is still required, because many simple return SUSPENDED state if anything went wrong (replacing the real error condition with SUSPENDED). A dedicated PROBABLE\_INVALID\_MSG return state is probably useful so that an output plugin can convey back that it consideres the message to be bad. On the other hand, this implies that the plugin must try to detect those, what means that the developer must think about all potential message-causes problems. That approach can be considered unreliable and as such it may be better not to provide such a dedicted state. + +\subsubsection{Handling of Failures} +In spite of the two different failure cases, different handling is needed for them. The action-based failure cases can and must be handled on the action level. As transactions abort when a failure occurs, support from the upper ``batch layer'' is necessary in order to handle resending batches of messages. + +For message-caused failure cases, the offending message must be found and then be discarded. A complexity here is that while a failure-causing message is being searched for, an action-based failure might occur. In that case, first the action-based failure condition must be solved, before the search for the problem message can continue. + +One approach might be that when the action-layer conveys back an action-caused failure (SUSPENDED), the batch layer knows that it simply needs to restart the full transaction (but not start an ``invalid message search''). If a message-based error condition is conveyed back, the batch system can not restart the full batch. Instead, it needs to enter search mode, where it creates partitions of the original batch, and calls itself recursively (at least in theory) on each of the subsets. + +Then, the same handling applies until either a failing message has been found or all messages have been successfully processed. Note that in the recursive step, action-based failures are recovered by full batch resubmits. This solves the above-mentioned complexity in a consistent way. + +If a binary-search-like method is used to detect failing records\footnote{This was originally suggested by David Lang.}, recursion may not really be an issue, as the recursion depth is limited to $\log_2 |B|$ where $B$ is the message batch. + +A message-caused failure can be rooted in one or more messages. One important question is if it is expected that the failure is caused by a single or multiple messages. Both is possible, so it is a question of probability. If we assume that it is more probable that a single messages causes the problems, it is useful to immediately return back to full batch submission of transactions once a problem-causing message has been identified. But then, if there are multiple problem-causing messages inside the batch, we may need many more iterations. + +If, on the other hand, we assume that it is more probable that multiple messages cause problems, it may make sense to keep resubmitting only subsets of the batch. However, then the performance is suboptimal if actually only one message was problematic. A solution might be to pick a compromise, e.g. first assume that a single message is problematic, but assume the opposite as soon as a second message with problems has been found. + +A potential algorithm for processing $n \le |B|$ messages from batch $B$ is described below. In the pseudocode, a ``processable'' message is one that neither is already committed nor had a permanent failure with this action. The term ``mpf'' means ``message permanent failure'' for this action (this will later be described in a batch state set). + +\begin{small} +\lstset{language=python} +\begin{lstlisting} +def submitBatch(B, n): + foreach processable message in + (first [at most] n messages of batch): + call processMessage + if action-caused failure: + retry full batch + if action-caused permanent failure: + mark all n messages as mpf + return + if auto-commit: + mark commited messages in batch as committed + if message-caused failure: + if n == 1: + mark message as mpf + return + else: + call submitBatch(B, n/2) + call submitBatch(B, n/2) +\end{lstlisting} +\end{small} + +After submitBatch() has completed, all messages are either committed or in mpf state. + +Note that an action-caused permanent failure occurs if an action-caused failure can not be resolved with the operator-configured number of retries. It will never occur if the user configured infinite retries. While an action is suspended, all calls will result in an action-caused permanent failure. Please keep in mind that these will be resubmitted to any backup actions inside the action unit, so the action's ability to cause permanent failure states is vital for a number of use cases (backup syslog server, to name just one). + +Batch processing inside an action unit thus can follow these strucuture: + +\begin{algorithm} +\caption{processBatch(B)} +\begin{algorithmic} +\FORALL{action $a$ in action unit} + \IF{execute action only on messages that failed before} + \STATE $n = |\text{messages in batch in mpf state}|$ + \STATE change mpf state back to ready + \ELSE + \STATE $n = |B \setminus \text{msgs with state discard}|$ + \STATE change all message states $\ne$ discard to ready + \ENDIF + \IF{$n >0$ } + \STATE call submitBatch(B, n) for action $a$ + \ENDIF +\ENDFOR +\end{algorithmic} +\end{algorithm} + +\paragraph{Why is it Important to differentiate the failure cases?} +This text originates from the mailing list and must be merged in. I provide it in the form it is, so it will not be forgotten (plus, it conveys the information). + +One may think that it is not necessary to differentiate between action-caused and message-caused failures. However, not doing so introduces subtle issues, because +then you either + +A) do not need the batch logic at all (because the action is configured for +infinite retries) + +Or + +B) you loose many messages if the action is not configured for infinite +retries and you have a longer-duration outage e.g. on a database server. +Let's say it is offline for a couple of hours, then you lose almost +everything in that period + +To prevent this, you need two different retry methods. + +One may argue that it is hard to differentiate between the two failure cases. This is correct. Buit I think it mostly depends on the quality of the output module. + +First of all, ``mostly'' implies that there may be some other cases, where it +really is impossible to differentiate between the two. In that case, I would +treat the issue as an action-caused failure. There are two reasons for this: + +1) rsyslog v3 currently does this always and not even a single person +complained about that so far. This is an empiric argument, and it does not +mean it caused problems. But it carries the co-notation that this seems not +to be too bad. + +2) If we would treat it as message-caused failure, we would no longer be able +to handle extended outages of destination systems, which I consider a vitally +important feature. + +When weighing the two, I know of lots of people who rely on 2), in sharp +contrast to knowig noone having problems with 1). So my conclusion is that it is +less problematic to define an otherwise undefinable failure reason to be +action-caused. Even more so as I assume this problem only exists in the +minority of cases. + +Now back to the quality of the output module: thinking about databases, their +API is usually very good at conveying back if there was a SQL error or a +connection abort. So while a SQL error may also be an indication of a +configuration problem, I would strongly tend to treat it is a being +message-caused. This is under the assumption that any reasonable responsive +admin will hopefully test his configuration at least once before turning it +into production. And config SQL errors should manifest immediately, so I +expect these to be fixed before a configuration runs in production. So it is +the duty of the output module to interpret the return code it received from +the API call and decide whether the failure is more likely action-caused or +message-caused. For database outputs, I would assume that it is always easy +to classify failures that must be action-caused, especially in the +dominating cases of failed network connections or failed servers. + +For other outputs it may not be as easy. But, for example, all stream network +outputs can detect a broken connection, so this also is a sure fit. + +For dynafiles, it really depends on how hard the output module is tries to differentiate +between the two failure cases. But I think you can go great length here, too. +Especially if you do not only look at the create() return code, but, iff a +failure occurs, you do more API calls to find out the cause. + +So I think the remaining problem is small enough to cause not too much issues +(and if so, they are unavoidable in any case). In conclusion, the two failure states are not only necessary, but can sufficiently sure enough be detected. + +\subsection{Random Topics} +I have begun to gather material from the mailing list in this section, because I feel it may be useful for others as well. Right now, the information is well hidden in the mailing list archives and there may be value in combining it all in one place. + +Due to the nature of this material, there is no specific organization between the subchapters and also formatting and language doesn't deny its rooting in the mailing list. + +\subsection{Reliability of Message Dequeueing} +A batch is actually dequeued when it is taken off a queue. So if at that point we +have a system power failure (for whatever reason), the messages are lost. +While the rsyslog engine intends to be very reliable, it is not a complete +transactional system. A slight risk remains. For this, you need to understand +what happens when the batch is processed. I assume that we have no sudden, +untrappable process termination. Then, if a batch cannot be processed, it is +returned back to the top of queue. This is not yet implemented, but is how +single messages (which you can think of an abstraction of a batch in the +current code) are handled. If, for example, the engine shuts down, but an +action takes longer than the configured shutdown timeout, the action is +cancelled and the queue engine reclaims the unprocessed messages. They go +into a special area inside the .qi file and are placed on top of the queue +once the engine restarts. + +The only case where this not work is sudden process termination. I see two +cases: + +a) a fatal software bug +We cannot really address this. Even if the messages were remaining in the +queue until finally processed, a software bug (maybe an invalid pointer) may +affect the queue structures at large, possibly even at the risk of total loss +of all data inside that queue. So this is an inevitable risk. + +b) sudden power fail +... which can and should be mitigated at another level + +One may argue that there also is + +c) admin error +e.g, kill -9 rsyslogd +Here a fully transactional queue will probably help. + +However, I do not think that the risk involved justifies a far more complex +fully transactional implementation of the queue object. Some risk always +remains (what in the disaster case, even with a fully transactional queue?). + +And it is so complex to let the messages stay in queue because it is complex +to work with such messages and disk queues. It would also cost a lot of +performance, especially when done reliably (need to sync). We would then need +to touch each element at least four times, twice as much as currently. Also, +the hybrid disk/memory queues become very, very complex. There are more +complexities around this, I just wanted to tell the most obvious. + +So, all in all, the idea is that messages are dequeued, processed and put +back to the queue (think: ungetc()) when something goes wrong. Reasonable +(but not more) effort is made to prevent message loss while the messages are +in unprocessed state outside of the queue. + +\paragraph{More reliable can actually be less reliable} +On the rsyslog mailing list, we had a discussion about how reliable rsyslog should be. It circles about a small potential window of message loss in the case of sudden fatal failure. Rsyslog can be configured to put all messages into a disk queue (instead of main memory), so these messages survive such a powerfail condition. However, messages dequeued and scheduled for processing during the power outage may be lost. + +I now consider a case where we have bursty UDP traffic and rsyslog is configured to use a disk-only queue (which obviously is much slower than an in-memory queue). Looking at processing speeds, the max burst rate is limited by using an ultra-reliable queue. To avoid using UDP messages, a second instance could be run that uses an in-memory queue and forwards received messages to the one in ultra-reliable mode (that is with the disk-only queue). So that second instance queues in memory until the (slower) reliable rsyslogd can now accept the message and put it into the reliable queue. Let's say that you have a burst of $r$ messages and that from these burst only $r/2$ can be enqueued (because the ultra reliable queue is so slow). So you lose $r/2$ messages. + +Now consider the case that you run rsyslog with just a reliable queue, one that is kept in memory but not able to cover the power failure scenario. Obviously, all messages in that queue are lost when power fails (or almost all to be precise). However, that system has a much broader bandwidth. So with it, there would never have been r messages inside the queue, because that system has a much higher sustained message rate (and thus the burst causes much less of trouble). Let's say the system is just twice as fast in this setup (I guess it usually would be *much* faster). Than, it would be able to process all r records. + +In that scenario, the ultra-reliable system loses $r/2$ messages, whereas the somewhat more "unreliable" system loses none - by virtue of being able to process messages as they arrive. + +Now extend that picture to messages residing inside the OS buffers or even those that are still queued in their sources because a stream transport blocked sending them. + +I know that each detail of this picture can be argued at length about. + +However, my opinion is that there is no "ultra-reliable" system in life, only various probabilities in losing messages. These probabilities often depend on each other, what makes calculating them very hard to impossible. Still, the probability of message loss in the system at large is just the product of the probabilities in each of its components. And reliability is just the inverse of that probability. + +This is where *I* conclude that it can make sense to permit a system to lose some messages under certain circumstances, if that influences the overall probability calculation towards the desired end result. In that sense, I tend to think that a fast, memory-queuing rsyslogd instance can be much more reliable compared to one that is configured as being ultra-reliable, where the rest of the system at large is badly influenced by this (the scenario above). + +However, I also know that for regulatory requirements, you often seem to need to prove that a system may not lose messages once it has received them, even at the cost of an overall increased probability of message loss. + +My view of reliability is much the same as my view of security: there is no such thing as "being totally secure", you can just reduce the probability that something bad happens. The worst thing in security is someone who thinks he is "totally secure" and as such is no longer actively looking at potential issues. + +The same I see for reliability. There is no thing like "being totally reliable" and it is a really bad idea to think you could ever be. Knowing this, one may begin to think about how to decrease the overall probability of message loss AND think about what rate is acceptable (and what to do with these cases, e.g. "how can they hurt"). + +\paragraph{Different Use Cases} +As David Lang pointed out, there exist different use cases for different levels of reliability. Most importantly, there exist use cases that do not demand very high throughput but rather ultra-realiability of the queue system. Here, ultra-reliability is just another word for the queue being of ``audit-grade''. Even if the queue provides audit-grade, the overall system is only then of audit-grade when all other components - most notably the transport protocols spoken by the inputs and outputs - are also of audit-grade. Most importantly, this means that an audit-grade system purely based on the IETF syslog protocol series can not be build. + +Used together with truly reliable protocols \emph{and} senders that block processing until a final acknowledgement has been received, an audit-grade system can potentially build based on rsyslog. To do so, an audit-grade queue subsystem is required, which is not present in releases less than 4.1.? (most importantly, v2 and v3 do not provide this capability). + +\subsection{Audit-Grade Queue Operations} +\subsubsection{Perquisites} +Audit-grade queue operations certain perquisites: +\begin{itemize} +\item rsyslog engine is of version 4.1.? or greater +\item disk-only queue type +\item checkpoint interval set to 1 +\item queue is configured to not permit losing any messages\footnote{The queue has several settings that can be used to fine-tune situations in which it may discard messages intentionally. All of these must be turned off. Most importantly, that means the producer is blocked for an infinite time if the queue is full.} +\item queue consumer must also be of audit-grade +\end{itemize} +Only when these prequisites are met, queue operation can be considered of being audit-grade. Note that when message loss in case of sudden fatal failure and similar incidents is acceptable, neither disk-only queues nore a checkpoint interval of 1 is necessary. Such a configuration can also be build with rsyslog v3, which is up to that level. + +Note that in the sections below we describe the implementation in broader terms. Most importantly, we do not restrict ourselves to disk-only queue storage drivers. This is important, because it simplifies design and opens the capability to introduce new, possibly faster-performing, queue storage drivers in the future. + +But it is important to keep in mind that a concrete queue is only of audit-grade if it matches all the perquisites given here, most importantly with the right configuration. + +\subsubsection{Implementation Alternatives} +Messages, or more precisely objects\footnote{While rsyslog deals with messages, the queue is designed to handle any type of thing that is represented as an rsyslog object. This is considered useful as queues may at some time contain other things than just messages, so we keep it generic.}, are enqueued by the queue producer (either an input module or the main message queue's consumer). The enqueue operation is completed only when the message has been successfully accepted by the queue storage driver. Then and only then the producer is permitted to remove the object from its own storage system. A rough sketch is given in algorithm \ref{alg_q_enq}. + +\begin{algorithm} +\caption{enqueueObject($o$)} +\begin{algorithmic} +\label{alg_q_enq} +\STATE lock queue mutex +\WHILE{queue is not ready for enqueue} + \STATE wait on queue to become ready +\ENDWHILE +\STATE call queue store driver to add $o$ +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +The dequeue-operation is more complex. We must ensure that each object stays in the queue until it is finally processed. Hereby, an object is finally processed, when processing of it has been completed. Remember that to enhance performance, objects are dequeued in batches of many. So at any given time, multiple messages may be processed, but not necessarily have finally completed doing so. If another worker thread then tries to obtain a new batch for processing, those ``in-process'' message must not be handed out a second time. Also, if a sudden fatal failure occurs during processing, queue operation must restart at the point of last commit. This means that all ``in-process'' messages need to be changed back to ``no processed'' state and be restarted again. In those cases the (acceptable) slight message duplication can occur. + +In our design, we differentiate between ``logical'' and ``physical'' dequeuing of batches. If a batch is generated for processing, it is logically dequeued --- in the sense that no other batch generating request will be able to receive another copy of these messages. If no exceptional situation happens, those messages will be processed and thus can be considered consumed under normal circumstances. + +However, actual deletion from the physical queue storage happens only after the batch is fully processed. At this point, all objects have been acknowledged by their destinations, which now have the responsibility for the object's survival. Consequently, we can delete them from the queue store. This process is considered the ``physical'' dequeue of the object. + +In order to find some simpler terms, we will call the logical dequeue operation just ``dequeue'' and the physical dequeue operation ``delete''. This is consistent with all previous work on rsyslog and thus probably leads to the least surprise when reading older source code and documentation. + +A first idea for a deletion is given in algorithm \ref{alg_pdeq_batch_1} (remember that $O(b)$ contains all objects within the given batch $b$, this is \emph{not} $O$-notation and should probably in the future be replaced by something else). + +\begin{algorithm} +\caption{deleteBatch($b$), first approach} +\begin{algorithmic} +\label{alg_pdeq_batch_1} +\STATE lock queue mutex +\FORALL{$o \in O(b)$} + \STATE find $o$ in queue storage + \STATE remove $o$ and keep queue structures intact +\ENDFOR +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +This algorithm is simple, but requires searching the queue store for the object to be deleted -- a potentially lengthy operation. However, we can improve the searching process if we know more about the inner structure of batch objects. It seems appropriate to dequeue objects in queue-sequential order. A drawback of doing so is that we must prevent other worker threads from trying to dequeue concurrently. This is not really a drawback. We need to guard dequeue operations by a mutex in any case, because otherwise internal structures can not be kept consistent. Practical experience and testing have shown that many small dequeue operations cause a lot of locking contention and as such badly affect performance. So it actually is a welcome enhancement to aquire the queue lock only once for the whole batch dequeue operation. As dequeing is a comperatively fast operation, the lock is not held for extended periods of time. + +A first approach to this functionality is shown in algorithm \ref{alg_ldeq_batch_1}. Note that $C_{mBatch}$ is the configured maximum number of elements inside a batch, $i$ is an index to address the objects inside the batch. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers.jpeg} +\end{center} +\caption{\textbf{Queue Store Pointers}: boxes represent queue entries, colored boxes entries with objects. Objects in green are unprocessed, in blue are dequeued but not deleted and those in gray have already been deleted. White indicates not yet used entries. Gray objects may be overwritten at any time. Their entries are actually free, we have used the gray color primarily to indicate there once existed objects. Each queue pointer points to the next entry to process.} +\label{fig_queue_ptr} +\end{figure} + +\begin{algorithm} +\caption{dequeueBatch($b$)} +\begin{algorithmic} +\label{alg_ldeq_batch_1} +\STATE lock queue mutex +\STATE $0 \to i$ +\WHILE{queue non-empty and $i < C_{mBatch}$} + \STATE obtain next obj $o$ from queue store + \STATE advance logical dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +A key concept is somewhat hidden in \marginpar{queue pointers} \emph{advance logical dequeue position}. Each queue store is purely sequential, with objects being enqueued at one ``end'' of the store and dequeued at the other. Of course, each queue store has only finite capacity, but we ignore this to explain the overall picture. A queue can be implemented by two pointers: one that points to the tail of the queue, where new messages are enqueued and one that points to the head of it, where new messages are dequeued. The idea is now to duplicate the dequeue pointer and split it into one for (logical) dequeue and one for deletion. Figure \ref{fig_queue_ptr} shows this three-pointer approach. Now, we can simple advance either the dequeue or deletion pointer, depending on operation, and do not need to find the first dequeue position inside the queue store. The dequeue pointer always points at it. This mode can be implemented with all currently existing queue storage drivers (but the sequential disk driver may need to use a second file handle or stream object instead of two pointers). + +This makes an efficient implementation of algorithm \ref{alg_ldeq_batch_1} possible: when it logically dequeues, it just needs to advance the dequeue pointer. So the algorithm executes in $O(n)$ time where $n$ specifies the number of elements to dequeue with an upper bound of $C_{mBatch}$. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers2.jpeg} +\end{center} +\caption{\textbf{Physically Dequeueing Messages}: In this sample, we have two batches. With multiple workers, they may be deleted in any order.} +\label{fig_queue_ptr_deq} +\end{figure} + +Furthermore, we can also improve algorithm \ref{alg_pdeq_batch_1}: Consider that each batch is logically dequeued as an atomic operation. That means all batch objects form a sequential subset of the queue. Figure \ref{fig_queue_ptr_deq} shows the situation when two batches have been dequeued. So the costly ``find'' operation now needs to be carried out only once at the beginning of the batch. As all other objects are sequential, once we have found the batch begin inside the queue, we can simply delete the $|b|$ elements in queue-sequential order after it. So the cost of the find operation can be reduced from $O(|b|)$ to $O(1)$. + +We can even reduce the remaining cost of the find operation. If the batch to be deleted is right at the queue's head (as is ``B1'' in the figure), the ``find'' immediately terminates with the first element and incurs no cost at all. The situation is different if the batch is not at the queue head, ``B2'' is an example for that (assuming that ``B1'' has not yet been dequeued). We would now still need to search over the objects that are not part of the batch and can then finally get to the object at the head of the batch in question. For queue storage drivers that support random access to queue elements, storing a simple pointer to the batches' queue head element further improves the situation and enables $O(1)$ access to the queue element. This is indicated by the dotted lines in figure \ref{fig_queue_ptr_deq}. Once the head of the queue has been found, two things can happen (depending on the capabilities of the queue storage driver): + +\begin{enumerate} +\item the head element can be flagged as ``this and next $n$ elements are deleted'' +\item all elements are actually deleted +\end{enumerate} + +Note that a mixed form is also possible (and probably useful for our \emph{singly} linked list storage driver: there, some $n'$ elements be actually deleted and the head element is flagged as ``this and next $n - n'$ elements are deleted''. Note that in the linked-list case, all but the first elements can be deleted with ease\footnote{It can be considered to change from a singly-linked list to a doubly-linked list, if the benefit outweighs the extra effort required.}, so probably just the head would stay inside the queue. Note that removing elements off the queue, where possible, is useful because it frees resources. On a busy system, freeing messages as soon as possible can prevent message loss (in non-audit-grade setup) or system slowdown. So it should be done when possible. + +If we have a purely sequential queue storage driver (currently the sequential disk driver), finding and updating the head element is not an option. Even in this case, we can observe that the batch at the actual deletion pointer will eventually be submitted for deletion. So a route to take is to create a list of elements that can be deleted as soon as the physical dequeue pointer reaches any of these elements. We call this the \marginpar{to-delete list}``to-delete list''. To facilitate processing, this list must be ordered in sequence of dequeing. This information may not be available from the storage subsystem itself, but it can easily be generated. To do so, a strictly monotonically increasing counter is kept with each logical dequeue operation and stored as part of the batch\footnote{As this must be done via the usual computer-implemented modular arithmetic, we must be careful that we do not see repetion of values because of overflows. Each day has $60 \cdot 60 \cot 24 = 86,400$ seconds (ignoring the subleties of UTC). Now let's assume that we have a moderately-busy system with 1,000 messages per second. We further assume, to be on the save side, that each message is processed inside its own batch. So we have $86,400,000$ batches per day. If we now use a typical $32$-bit integer for generating the batch IDs, we the unique range will be used up after +$$\frac{2^{32}}{8640000} \approx 497 \text{ days}$$ +days of uninterrupted rsyslog operation. While this sounds somewhat save, it goes down to approximately 10 days of messages are submitted at rate of 50,000 messages per second (which is high, but not unheared of). So it is strongly advised to use 64 bits, which we consider to be save, because for our 1,000 messages per second the range would be exhausted only after +$$\frac{2^{64}}{8640000} \approx 2.135 \cdot 10^{11} \text{ days}$$ +which equals approximately $584,500,000$ \emph{years}. So even at a rate of one million messages per second, the range would be sufficient for over 500,000 years of continuos operations -- that should be far sufficient.} +An example: let us assume that ``B2'' was submitted for deletion first. Then, the head of ``B2'' is not at the queue's delete pointer. As such, no action can be carried out immediately. So the batch head pointer is stored into a ``to be deleted'' list. Processing continues. Some time later, batch ``B1'' is submitted for deletion. Now, the head pointer is at the head of the delete list, as such all batch elements are dequeued. Then, the ``to be deleted'' list is checked, and ``B2'' is found in it. Now, ``B2'' is at the head of the (new) deletion pointer and can also be removed. So, ultimately, all messages are physically dequeued. This is more formally describe in algorithm \ref{alg_phys_deq_seq_store}. In that pseudocode, we made a simplification by always putting the to be deleted batch in the ``to-delete'' list, which then enables us to use somewhat more generic code to carry out the work. + +Note that there is a price to pay for deletions via the ``to-delete'' list: if a sudden fatal failure happens during processing, the set of duplicate messages is increased. For example, if a fatal failure happens after ``B2'' has been fully processed and scheduled for deletion, but \emph{before ``B1'' is also submitted for deletion}, ``B2'' will be reprocessed after recovery. This would not happen if ``B2'' would have been removed from the queue. + +\begin{algorithm} +\caption{deleteBatch($b$)} +\begin{algorithmic} +\label{alg_phys_deq_seq_store} +\REQUIRE queue mutex is locked by caller +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.deletePtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.deletePtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\end{algorithmic} +\end{algorithm} + +\paragraph{Warp-Up of Queue Delete Operations} +When evaluating which route to take, the ``to-delete'' list approach looks elegant for all cases. The negative side effect of potentially increased message duplication currently does not even exist: today, the sequential disk queue storage driver permits only a single worker thread and thus there always will be only one thread at a time. Even if we remove that limitation, message duplication could not be avoided, as stated in the algorithm description above. What remains are the other queue storage drivers. However, they operate in-memory, so message duplication will not happen simply because all messages will be lost on sudden fatal failure. The advantage of limited message duplication only exists in the so-far hypothetical case of a random-access, audit-grade disk queue storage driver. Thus, the decision could be postponed unless that happens (if it ever does). + +From a code complexity point of view, the ``to-delete'' list approch is definitely advantagous. Not only because of the reduced number of algorithms required. We also do not need to maintain unique batch IDs and all the logic associated with them. + +The other aspect to look at is memory consumption. Assuming that we delete the actual objects, just not their containers inside the queue, extra memory consumption is not really that worse. More importantly, currently only the linked-list queue storage driver can benefit at all, because it is the only driver capable of deleting queue entries in mid-queue. All others, including the array memory driver, do not have this capability. + +From a performance point of view, the ``to delete'' list approach looks approximately as good as the others, with some mild better performance for some storage drivers for a non-``to delete'' list approach. This can be mitigated, especially if the potentially somewhat-costly maintenance of the ``to-delete'' list is slightly optimized and the algorithm actually checks if the to be deleted batch is right at the queue's delete pointer position. The improved code simplicity, together with current CPU's code caching, may even result in an otherwise not expected speedup. + +In conclusion, we will implement the ``to-delete'' list approach on the queue layer (above the queue storage drivers). However, we will leave the window open to permit overwriting it with queue storage driver specific functionality. How to do this will not be specified now, as there is currently no need and we do not even know if there ever will be. However, we retain the discussion on the various modes as well as the relevant algorithmic discussions and data structurs inside this paper so that it is readily available should need arise. We also think this is important so that everybody later knows that the decision was made based on good argument and not by accident (we consider this useful in another design enhancement attempt). + +\paragraph{Processing Sequence} Looking at the processing sequence, we notice that always objects are dequeued, then processed and then deleted. Then, the whole process starts again. In particular, this meanss that after the previous batch has been deleted, the next batch will be dequeued. Now consider that we need to have exclusive access to the queue for both of these operations. As such it seems natural to combine this into a single step, further reducing potential locking contention. + +Note that a side-effect of this approach is that messages can be deleted only when a new batch is dequeued. With current design, this means that at least one message must reside inside the queue. Otherwise, the last batch will not be deleted. However, this something that can (and must!) be solved on the queue worker layer, in that it deletes a batch when the queue is empty. + +This leads us to the implementation of dequeueBatch() and deleteBatch() shown in algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. Note that $l$ is a flag variable that indicates if the queue is already locked. + +\begin{algorithm} +\caption{dequeueBatch($b$): final version} +\begin{algorithmic} +\label{alg_deq_batch_final} +\STATE lock queue mutex +\STATE call deleteBatch(b, 1) +\STATE $0 \to i$ +\WHILE{queue non-empty and $i < C_{mBatch}$} + \STATE obtain next obj $o$ from queue store + \STATE advance dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + + +\begin{algorithm} +\caption{deleteBatch($b, l$): final version} +\begin{algorithmic} +\label{alg_del_batch_final} +\IF{queue not yet locked (test via $l$)} + \STATE lock queue mutex +\ENDIF +\FORALL{objects $o$ in $b$} + \STATE destruct $o$ +\ENDFOR +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.deletePtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.deletePtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\IF{queue not yet locked (test via $l$)} + \STATE unlock queue mutex +\ENDIF +\end{algorithmic} +\end{algorithm} + +\subsubsection{Queue Stores} +Currently, rsyslog supports three different types of queue store drivers: + +\begin{itemize} +\item memory array +\item memory linked list +\item disk sequential file +\end{itemize} + +They all provide an abstracted sequential queue store as shown in figure \ref{fig_queue_ptr} on page \pageref{fig_queue_ptr}. + +Obviously, some differences exist. Most importantly, the disk sequential file driver does \emph{not} support more than one queue worker thread (in order to prevent excessive disk activity and the subtle issues with rewriting parts of sequential files). So if this driver is used, the queue automatically limits itself to a maximum of one worker thread (even if user configuration settings + +Different queue store drivers have different properties: + +\begin{tabular}{|l||l|l|l|}\hline + & array & linked list & seqential file \\ \hline +pointer type & integer index & memory address & file number and \\ + & & & offset within file \\ \hline +physical access & random & random & sequential \\ \hline +remove middle & no & yes & no \\ +elements & & & \\ \hline +access to $n$-th& $O(1)$, index:& $O(n)$, follow & not supported \\ +element & $n \mod C_{mMsg}$ & pointer links & \\ \hline +speed & fastest & fast & slow \\\hline +mem overhead & large & some & almost none \\\hline +reliability & reliable & reliable & audit-grade\footnote{if configured correctly}\\ +\hline +\end{tabular} + +\subsubsection{Implementation} +The actual implementation will be based on algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. The rsyslog v3 queue storage driver will be extended one additional method, which permits non-destructive dequeueing of elements. As such, the driver now has the $qAdd()$, $qDeq()$, and $qDel()$ entry points (together with the usual construction and destruction entry points). The queue drivers must support the three pointers for enqueue, dequeue and delete. The ``to-delete'' list will be maintained on the upper queue layer (and not the queue driver layer). This functionality will be optimized so that if a batch to delete is right at the queue's delete pointer, it will immediatly be deleted and not be sent to the ``to-delete'' list. This is especially important with the sequential disk driver, as the condition here always is true (and thus the driver can pretend this in the relevant API without even comparing any pointers -- what would otherwise quite complicated in this driver. + +The full list of the queue store driver interface is: + +\paragraph{qConstruct} Initializes the queue store. + +\paragraph{qDestruct} Destructs the queue store, including all messages that may still be present in it. + +\paragraph{qAdd} Enqueue a new object into the queue. Note that this entry point must only be called when the queue is non-full. + +\paragraph{qDeq} Non-destructive dequeue of the object at queue head. Dequeue pointer is advanced. + +\paragraph{qDel} Delete the object at queue head. Delete pointer is advanced. + +Disk queue store drivers may support additional internal functions. However, they should not be exposed to the rest of the queue subsystem. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{queue_msg_state.jpeg} +\end{center} +\caption{Logical Message States during Queue Processing} +\label{fig_queue_msg_state} +\end{figure} + +Figure \ref{fig_queue_msg_state} shows a logical message state diagram during queue processing. There is no actual state variable, but rather the processing flow demands these state. Note that the state transition from ``dequeued'' to ``queued'' only happens after a fatal failure and a successful system recovery. So this is a rather exceptional case. + +Another subtle issue is that we now need two different queue size counters: one for seeing when the queue is physically full and one for detecting when there are no more messages to be dequeued. + +As a simplification, support for ungetting objects can be removed (as objects never leave the queue), what also means that cancel-processing is probably less complex. + +\paragraph{Sequential Disk Queue Store Driver} +The enequeue, deqeueue and delete pointers must be implemented via three stream objects. Most importantly, the dequeue stream must be configured not to delete files when it closes them. A side-effect of this implementation is that data is actually read twice, once to actually obtain it and a second time to delete it. This could only be avoided by an overall redesign on how the disk queue works. + +\subsubsection{Checkmarks} +The following things need to be verified in the actual implementation. + +\paragraph{Queue Full} +Is it possible to set an infinte timeout on queue full condition during enqueue? If not, we must provide it. + +\paragraph{Termination the Queue} +If we cancel a worker, we need to start from the physical dequeue pointer and pull everything that is not scheduled for deletion - NOT from the logical dequeue pointer. + +\paragraph{Failed Messages} +If a message fails on a detached action queue, no backup processing is available (because we detect the failure at a point where the message is already considered processed from the main queue's point of view. We need address this and have two options: + + +I see two approaches at handling this: + +a) we enable an action to configure a backup file that shall receive all +message permanent failures. This is simple (not only to implement but to +configure and understand) + +b) we push the failed message back to the main queue, but with an indication +that it failed in an action. This is harder to implement and most importantly +harder to understand/configure, but more flexible + +\section{Network Stream Subsystem} +The idea of network streams was introduced when we implemented RFC5425 (syslog over TLS) in 2008. The core idea is to encapsulate all stream-oriented network data transfer into a single transport layer and make the upper layers independent of actual transport being used. This is in line with the traditional layer approaches in communication systems. + +Under this system, the upper layer provides plugins to send and receive streams of syslog data. Framing is provided by the upper layer. The upper layer itself is integrated in input and output plugins, which then are used to provide application-level syslog message objects to and from the rsyslog core. To these upper layers, the netstream layer provides reliable and sequenced message delivery with much of the same semantics as a usual TCP stream. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{tls.jpeg} +\end{center} +\caption{Objects at the Network Stream Layer} +\label{fig_netstream_objects} +\end{figure} + +At the netstream layer, we have a small set of generic classes, which are used for setup of the drivers and driver parameters. This is a very thin layer, mostly a wrapper. Once an actual lower-level netstream driver has been loaded, all parameters are passed through to it. + +Please note that both in theory and practice netstream drivers may call back into different netstream drivers. For example, the GnuTLS RFC5425 driver loads and calls back into the plain tcp driver, simply because that driver provides part of the required functionality and there is no point in re-implementing it for GnuTLS. + +The netstream driver layer does not only provide read and write calls but supports i/o multiplexing. To do so, it offers an interface that follows select() semantics. That permits an upper-layer comonent to request being blocked unless some data arrives. Note that due to the subleties in TLS processing, the upper layer may be awoken while there is no upper-layer work to do. This will properly be indicated by the netstream subsystem, is not an error and must be accepted and poperly handled by the upper layer. + +Using the nestream layer, we do not need to modify the input and output plugins while at the same time we can add additional transport providers. One weak spot in this design is the current configuration process. With the current system, we need to provide one configuration statement per driver property and we need to hardcode this. So if a new driver would require new properties, we still would need to modify the upper layers. This is unfortunate, but the current config system does not provide for any better way to handle the situation. Once we are able to create a new config system, we will address this by providing the ability to pass a string of parameters onto the driver, which will then have the ability to parse its content. So once we do this, we need to modify the driver interface, but the end result would be a simlification. + +So far, only drivers for GnuTLS and plain tcp are provided. However, during the design of the layer we also looked at openssl and Mozilla Network Security Services as well as kept an eye on the needs of Kerberos. In theory, it should not be a major problem to write drivers for these systems (but it most probably still is a lot of work to do). + +A final note on Kerberos: in order to keep compatible with previous protocol handling and due to constraints in testing environment and knowledge, we still support Kerberos not via the netstream layer but via special extension into the input and output modules. That, too, is unfortunate, but given the current resources at hand, there is no alternative to handling in that way. We would be very interested in moving over Kerberos to a netstream driver and any volunteer would be very welcome. + +\section{Future Development} +This section covers topics that can not currently be developed, but where important thoughts came up in discussions. For obvious reasons, the section has brainstorming character. + +\subsection{Lock-Free Queuing} +On a very busy system, lock contention can limit performance. We should investigate ways to apply lock-free algorithms inside rsyslog. It is believed that at least for some scenarios, lock-free algorigthms can be applied with great benefit. To do so, we should introduce new queue modes, which will use very different semantics from what is described so far for the queue engine. Most importantly, in lock-free mode we will have limits on the number of producers and we will most probably not be able to guarantee audit-grade processing. The later is not a problem, because there are ample use cases that do not require audit-gradeness. + +\subsection{Audit-Grade High Performance Queue Storage Driver} +An audit grade driver must ensure that no message is lost, but should also be able to handle large workloads. The sequential disk driver does not support the later. + +An additional disk driver is envisioned with the properties like the linked list driver, but a reliable on-disk store. In particular, random access to queue elements is desired, which requires an addressing capability. + +A potential implementation requires a pre-formatted file. That file is organized in pages of $n$ bytes (e.g. 1K). The page index is used to address a queue item. If an item fits into 1K, it uses one page. If it is larger than 1K, consequtive pages are used to store the element. A page header must be present to indicate how many pages a single element is made up of. + +It may be noted that we could even improve performance by keeping part of the data in-memory. For audit-gradeness, it is required that upon enqueue the message is written to disk and only after final processing it needs to be removed. However, it is not forbidden to keep the same message in main memory. That way, the logical dequeue operation could be done one the in-memory representation. Only the physical dequeue would need to write to disk again. As such, we save one disk read out of three writes and one read otherwise required (so one can roughly say that we save one third of disk operations. + +Note that due to potential multi-pages messages we can not directly address individual elements, but we can reliably and quikly address elements whom's address we know (learned, for example, during logical dequeue). This is similar to the organization of the in-memory linked list. Actally, such a store \emph{is} a linked list implementation, just that memory is allocated on disk instead of in main memory. + +To further improve speed, object representation could be zipped before being written to a page. + +File Layout +Page 0: control structures (most importantyle queue pointers) (can make sense to store in a separate file, which could be moved to a dedicated disk subsystem - can potentially greatly reduce disk seek times). +Page 1 to n: actual object storage + +Algorithms \ref{alg_AuditGradeStoreEnqueue} and \ref{alg_AuditGradeStoreDelete} show how records are enqueued and deleted. Note that the delete part does not even need to read back the record. If we keep at last some records in-memory, the performance cost of ultra-reliable mode can actually comparatively low. Note that we may not even really need to commit data to the storage system in ``AuditGradeStoreDelete()'', because if a fatal failure occurs at this point, at worst message duplication may happen, what we have considered to be acceptable. + +\begin{algorithm} +\caption{AuditGradeStoreEnqueue($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreEnqueue} +\REQUIRE queue mutex is locked by caller +\STATE write $o$ to current enqueue location +\STATE update \& write queue structures [page 0] +\STATE sync all files touched +\STATE store $o$ in an in-memory structure (or a cache) +\end{algorithmic} +\end{algorithm} + +\begin{algorithm} +\caption{AuditGradeStoreDelete($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreDelete} +\REQUIRE queue mutex is locked by caller +\STATE update queue dequeue pointer \& write queue structures [page 0] +\STATE sync all files touched +\end{algorithmic} +\end{algorithm} + + +\end{document} diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html index cc2f7f38..63c186a3 100644 --- a/doc/dev_oplugins.html +++ b/doc/dev_oplugins.html @@ -144,19 +144,172 @@ array-passing capability not blindly be used.</b> In such cases, we can not guar plugin from segfaulting and if the plugin (as currently always) is run within rsyslog's process space, that results in a segfault for rsyslog. So do not do this. <h3>Batching of Messages</h3> -<p>With the current plugin interface, each message is passed via a separate call to the plugin. -This is annoying and costs performance in some uses cases (primarily for database outputs). -However, that's the way it (currently) is, no easy way around it. There are some ideas -to implement batching capabilities inside the rsyslog core, but without that the only -resort is to do it inside your plugin yourself. You are not prohibited from doing so. -There are some consequences, though: most importantly, the rsyslog core is no longer -intersted in messages that it passed to a plugin. As such, it will not try to make sure -the message is not lost before it was ultimately processed (because rsyslog, due to -doAction() returning successfully, thinks the message *was* ultimately processed). -<p>When the rsyslog core receives batching capabilities, this will be implemented in -a way that is fully compatible to the existing plugin interface. While we have not yet -thought about the implementation, that will probably mean that some new interfaces -or options be used to turn on batching capabilities. +<p>Starting with rsyslog 4.3.x, batching of output messages is supported. Previously, only +a single-message interface was supported. +<p>With the <b>single message</b> plugin interface, each message is passed via a separate call to the plugin. +Most importantly, the rsyslog engine assumes that each call to the plugin is a complete transaction +and as such assumes that messages be properly commited after the plugin returns to the engine. +<p>With the <b>batching</b> interface, rsyslog employs something along the line of +"transactions". Obviously, the rsyslog core can not make non-transactional outputs +to be fully transactional. But what it can is support that the output tells the core which +messages have been commited by the output and which not yet. The core can than take care +of those uncommited messages when problems occur. For example, if a plugin has received +50 messages but not yet told the core that it commited them, and then returns an error state, the +core assumes that all these 50 messages were <b>not</b> written to the output. The core then +requeues all 50 messages and does the usual retry processing. Once the output plugin tells the +core that it is ready again to accept messages, the rsyslog core will provide it with these 50 +not yet commited messages again (actually, at this point, the rsyslog core no longer knows that +it is re-submiting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 +messages were commited (before it failed), then only 10 would have been requeued and resubmitted. +<p>In order to provide an efficient implementation, there are some (mild) constraints in that +transactional model: first of all, rsyslog itself specifies the ultimate transaction boundaries. +That is, it tells the plugin when a transaction begins and when it must finish. The plugin +is free to commit messages in between, but it <b>must</b> commit all work done when the core +tells it that the transaction ends. All messages passed in between a begin and end transaction +notification are called a batch of messages. They are passed in one by one, just as without +transaction support. Note that batch sizes are variable within the range of 1 to a user configured +maximum limit. Most importantly, that means that plugins may receive batches of single messages, +so they are required to commit each message individually. If the plugin tries to be "smarter" +than the rsyslog engine and does not commit messages in those cases (for example), the plugin +puts message stream integrity at risk: once rsyslog has notified the plugin of transacton end, +it discards all messages as it considers them committed and save. If now something goes wrong, +the rsyslog core does not try to recover lost messages (and keep in mind that "goes wrong" +includes such uncontrollable things like connection loss to a database server). So it is +highly recommended to fully abide to the plugin interface details, even though you may +think you can do it better. The second reason for that is that the core engine will +have configuration settings that enable the user to tune commit rate to their use-case +specific needs. And, as a relief: why would rsyslog ever decide to use batches of one? +There is a trivial case and that is when we have very low activity so that no queue of +messages builds up, in which case it makes sense to commit work as it arrives. +(As a side-note, there are some valid cases where a timeout-based commit feature makes sense. +This is also under evaluation and, once decided, the core will offer an interface plus a way +to preserve message stream integrity for properly-crafted plugins). +<p>The second restriction is that if a plugin makes commits in between (what is perfectly +legal) those commits must be in-order. So if a commit is made for message ten out of 50, +this means that messages one to nine are also commited. It would be possible to remove +this restriction, but we have decided to deliberately introduce it to simpify things. +<h3>Output Plugin Transaction Interface</h3> +<p>In order to keep compatible with existing output plugins (and because it introduces +no complexity), the transactional plugin interface is build on the traditional +non-transactional one. Well... actually the traditional interface was transactional +since its introduction, in the sense that each message was processed in its own +transaction. +<p>So the current <code>doAction()</b> entry point can be considered to have this +structure (from the transactional interface point of view): +<p><pre><code> +doAction() + { + beginTransaction() + ProcessMessage() + endTransaction() + } + </code></pre> +<p>For the <b>transactional interface</b>, we now move these implicit <code>beginTransaction()</code> +and <code>endTransaction(()</code> call out of the message processing body, resulting is such +a structure: +<p><pre><code> +beginTransaction() + { + /* prepare for transaction */ + } + +doAction() + { + ProcessMessage() + /* maybe do partial commits */ + } + +endTransaction() + { + /* commit (rest of) batch */ + } +</code></pre> +<p>And this calling structure actually is the transactional interface! It is as simple as this. +For the new interface, the core calls a <code>beginTransaction()</code> entry point inside the +plugin at the start of the batch. Similarly, the core call <code>endTransaction()</code> at the +end of the batch. The plugin must implement these entry points according to its needs. +<p>But how does the core know when to use the old or the new calling interface? This is rather +easy: when loading a plugin, the core queries the plugin for the <code>beginTransaction()</code> +and <code>endTransaction()</code> entry points. If the plugin supports these, the new interface is +used. If the plugin does not support them, the old interface is used and rsyslog implies that +a commit is done after each message. Note that there is no special "downlevel" handling +necessary to support this. In the case of the non-transactional interface, rsyslog considers +each completed call to <code>doAction</code> as partial commit up to the current message. +So implementation inside the core is very straightforward. +<p>Actually, <b>we recommend that the transactional entry points only be defined by those +plugins that actually need them</b>. All others should not define them in which case +the default commit behaviour inside rsyslog will apply (thus removing complexity from the +plugin). +<p>In order to support partial commits, special return codes must be defined for +<code>doAction</code>. All those return codes mean that processing completed successfully. +But they convey additional information about the commit status as follows: +<p> +<table border="0"> +<tr> +<td valign="top"><i>RS_RET_OK</i></td> +<td>The record and all previous inside the batch has been commited. +<i>Note:</i> this definition is what makes integrating plugins without the +transaction being/end calls so easy - this is the traditional "success" return +state and if every call returns it, there is no need for actually calling +<code>endTransaction()</code>, because there is no transaction open).</td> +</tr> +<tr> +<td valign="top"><i>RS_RET_DEFER_COMMIT</i></td> +<td>The record has been processed, but is not yet commited. This is the +expected state for transactional-aware plugins.</td> +</tr> +<tr> +<td valign="top"><i>RS_RET_PREVIOUS_COMMITTED</i></td> +<td>The <b>previous</b> record inside the batch has been committed, but the +current one not yet. This state is introduced to support sources that fill up +buffers and commit once a buffer is completely filled. That may occur halfway +in the next record, so it may be important to be able to tell the +engine the everything up to the previouos record is commited</td> +</tr> +</table> +<p>Note that the typical <b>calling cycle</b> is <code>beginTransaction()</code>, +followed by <i>n</i> times +<code>doAction()</code></n> followed by <code>endTransaction()</code>. However, if either +<code>beginTransaction()</code> or <code>doAction()</code> return back an error state +(including RS_RET_SUSPENDED), then the transaction is considered aborted. In result, the +remaining calls in this cycle (e.g. <code>endTransaction()</code>) are never made and a +new cycle (starting with <code>beginTransaction()</code> is begun when processing resumes. +So an output plugin must expect and handle those partial cycles gracefully. +<p><b>The question remains how can a plugin know if the core supports batching?</b> +First of all, even if the engine would not know it, the plugin would return with RS_RET_DEFER_COMMIT, +what then would be treated as an error by the engine. This would effectively disable the +output, but cause no further harm (but may be harm enough in itself). +<p>The real solution is to enable the plugin to query the rsyslog core if this feature is +supported or not. At the time of the introduction of batching, no such query-interface +exists. So we introduce it with that release. What the means is if a rsyslog core can +not provide this query interface, it is a core that was build before batching support +was available. So the absence of a query interface indicates that the transactional +interface is not available. One might now be tempted the think there is no need to do +the actual check, but is is recommended to ask the rsyslog engine explicitely if +the transactional interface is present and will be honored. This enables us to +create versions in the future which have, for whatever reason we do not yet know, no +support for this interface. +<p>The logic to do these checks is contained in the <code>INITChkCoreFeature</code> macro, +which can be used as follows: +<p><pre><code> +INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); +</code></pre> +<p>Here, bCoreSupportsBatching is a plugin-defined integer which after execution is +1 if batches (and thus the transational interface) is supported and 0 otherwise. +CORE_FEATURE_BATCHING is the feature we are interested in. Future versions of rsyslog +may contain additional feature-test-macros (you can see all of them in +./runtime/rsyslog.h). +<p>Note that the ompsql output plugin supports transactional mode in a hybrid way and +thus can be considered good example code. + +<h2>Open Issues</h2> +<ul> +<li>Processing errors handling +<li>reliable re-queue during error handling and queue termination +</ul> + + + <h3>Licensing</h3> <p>From the rsyslog point of view, plugins constitute separate projects. As such, we think plugins are not required to be compatible with GPLv3. However, this is diff --git a/doc/expression.html b/doc/expression.html index 9e37cb7a..c401d9ab 100644 --- a/doc/expression.html +++ b/doc/expression.html @@ -1,17 +1,22 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head> -<meta http-equiv="Content-Language" content="en"><title>Expressions</title></head> +<meta http-equiv="Content-Language" content="en"> +<title>Expressions in rsyslog</title></head> <body> -<a href="rsyslog_conf_filter.html">back</a> -<h1>Expressions</h1> +<a href="rsyslog_conf_filter.html">back to rsyslog filter conditions</a> +<h1>Expressions in rsyslog</h1> <p>Rsyslog supports expressions at a growing number of places. So -far, they are supported for filtering messages.</p><p>Expression support is provided by RainerScript. For now, please see the formal expression definition in <a href="rainerscript.html">RainerScript ABNF</a>. It is the "expr" node.</p><p>C-like comments (/* some comment */) are supported <span style="font-weight: bold;">inside</span> the expression, but not yet in the rest of the configuration file.</p><p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +far, they are supported for filtering messages.</p> +<p>Expression support is provided by RainerScript. Please see the +<a href="rainerscript.html">RainerScript documentation</a> for more details.</p> +<p>C-like comments (/* some comment */) are supported <b>inside</b> the expression, +but not yet in the rest of the configuration file.</p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the -<a href="http://www.rsyslog.com/">rsyslog</a> -project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body></html> diff --git a/doc/imklog.html b/doc/imklog.html index 9166bae6..5bfab5ce 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -34,9 +34,9 @@ handled. The default is "off", in which case these messages are ignored. Switch it to on to submit non-kernel messages to rsyslog processing.<span style="font-weight: bold;"></span></li> <li><span style="font-weight: bold;"></span>$DebugPrintKernelSymbols -(imklog) [on/<b>off</b>]<br> +[on/<b>off</b>]<br> Linux only, ignored on other platforms (but may be specified)</li> -<li>$klogSymbolLookup (imklog) [on/<b>off</b>] -- +<li>$klogSymbolLookup [on/<b>off</b>] -- disables imklog kernel symbol translation (former klogd -x option). NOTE that this option is counter-productive on recent kernels (>= 2.6) because the kernel already does the symbol translation and this option breaks the information.<br> @@ -44,10 +44,19 @@ kernel already does the symbol translation and this option breaks the informatio it except if you have a very good reason. If you have one, let us know because otherwise new versions will no longer support it.<br> Linux only, ignored on other platforms (but may be specified)</li> -<li>$klogUseSyscallInterface (imklog) [on/<b>off</b>] +<li><b>$klogConsoleLogLevel</b> [<i>number</i>] +(former klogd -c option) -- sets the console log level. If specified, only messages with +up to the specified level are printed to the console. The default is -1, which means that +the current settings are not modified. To get this behavior, do not specify +$klogConsoleLogLevel in the configuration file. Note that this is a global parameter. Each time +it is changed, the previous definition is re-set. The one activate will be that one that is +active when imklog actually starts processing. In short words: do not specify this +directive more than once! +<br><b>Linux only</b>, ignored on other platforms (but may be specified)</li> +<li><b>$klogUseSyscallInterface</b> [on/<b>off</b>] -- former klogd -s option<br> Linux only, ignored on other platforms (but may be specified)</li> -<li>$klogSymbolsTwice (imklog) [on/<b>off</b>] -- +<li>$klogSymbolsTwice [on/<b>off</b>] -- former klogd -2 option<br> Linux only, ignored on other platforms (but may be specified)<br style="font-weight: bold;"> </li> @@ -69,7 +78,7 @@ is needed to start pulling kernel messages.<br> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2008-2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/imrelp.html b/doc/imrelp.html index 53826ac2..2cf9c1f7 100644 --- a/doc/imrelp.html +++ b/doc/imrelp.html @@ -35,6 +35,9 @@ Starts a RELP server on selected port</li> <b>Caveats/Known Bugs:</b> <ul> <li>see description</li> +<li>To obtain the remote system's IP address, you need to have at least +librelp 1.0.0 installed. Versions below it return the hostname instead +of the IP address.</li> </ul> <p><b>Sample:</b></p> <p>This sets up a RELP server on port 20514.<br> diff --git a/doc/imtcp.html b/doc/imtcp.html index 9ea7efa1..422bbd55 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -1,27 +1,29 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html><head> -<meta http-equiv="Content-Language" content="en"><title>TCP Syslog Input Module</title></head> +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>TCP Syslog Input Module</title> +</head> + <body> -<a href="rsyslog_conf_modules.html">back</a> +<a href="rsyslog_conf_modules.html">back to rsyslog module overview</a> <h1>TCP Syslog Input Module</h1> <p><b>Module Name: imtcp</b></p> -<p><b>Author: </b>Rainer Gerhards -<rgerhards@adiscon.com></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Multi-Ruleset Support: </b>since 4.5.0 and 5.1.1 <p><b>Description</b>:</p> <p>Provides the ability to receive syslog messages via TCP. -Encryption can be provided by using <a href="rsyslog_stunnel.html">stunnel</a> -(an alternative is the use -the <a href="imgssapi.html">imgssapi</a> -modul).</p> -<p>Multiple receivers may be configured by -specifying +Encryption is natively provided by selecting the approprioate network stream driver and +can also be provided by using <a href="rsyslog_stunnel.html">stunnel</a> +(an alternative is the use the <a href="imgssapi.html">imgssapi</a> module).</p> +<p>Multiple receivers may be configured by specifying $InputTCPServerRun multiple times. This is available since version 4.3.1, earlier versions do NOT support it. </p> <p><b>Configuration Directives</b>:</p> <ul> -<li>$InputTCPServerAddtlFrameDelimiter <Delimiter><br> +<li><b>$InputTCPServerAddtlFrameDelimiter <Delimiter></b><br> This directive permits to specify an additional frame delimiter for plain tcp syslog. The industry-standard specifies using the LF character as frame delimiter. Some vendors, notable Juniper in their NetScreen products, use an invalid frame delimiter, in Juniper's @@ -41,20 +43,37 @@ very limited interest in fixing this issue. This directive <b>can not</b> fix th That would require much more code changes, which I was unable to do so far. Full details can be found at the <a href="http://www.rsyslog.com/Article321.phtml">Cisco tcp syslog anomaly</a> page. -<li>$InputTCPServerRun <port><br> +<li><b>$InputTCPServerDisableLFDelimiter</b> <on/<b>off</b>> (available since 5.5.3)<br> +Industry-strandard plain text tcp syslog uses the LF to delimit syslog frames. However, +some users brought up the case that it may be useful to define a different delimiter and +totally disable LF as a delimiter (the use case named were multi-line messages). This mode +is non-standard and will probably come with a lot of problems. However, as there is need +for it and it is relatively easy to support, we do so. Be sure to turn this setting to +"on" only if you exactly know what you are doing. You may run into all sorts of troubles, +so be prepared to wrangle with that! +<li><b>$InputTCPServerNotifyOnConnectionClose</b> [on/<b>off</b>] (available since 4.5.5)<br> +instructs imtcp to emit a message if the remote peer closes a connection.<br> +<b>Important:</b> This directive is global to all listeners and must be given right +after loading imtcp, otherwise it may have no effect.</li> +<li><b>$InputTCPServerRun</b> <port><br> Starts a TCP server on selected port</li> -<li><ul><li>$InputTCPMaxSessions <number></li></ul> -Sets the maximum number of sessions supported</li><li>$InputTCPServerStreamDriverMode <number><br> +<li><b>$InputTCPMaxListeners</b> <number><br> +Sets the maximum number of listeners (server ports) supported. Default is 20. This must be set before the first $InputTCPServerRun directive.</li> +<li><b>$InputTCPMaxSessions</b> <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> +<li><b>$InputTCPServerStreamDriverMode</b> <number><br> Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specifc.</li> -<li>$InputTCPServerInputName <name><br> +<li><b>$InputTCPServerInputName</b> <name><br> Sets a name for the inputname property. If no name is set "imtcp" is used by default. Setting a name is not strictly necessary, but can be useful to apply filtering based on which input the message was received from. -<li>$InputTCPServerStreamDriverAuthMode <mode-string><br> -Sets the authentication mode for the currently selected <a href="netstream.html">network stream driver</a>. <mode-string> is driver specifc.</li><li>$InputTCPServerStreamDriverPermittedPeer <id-string><br> +<li><b>$InputTCPServerStreamDriverAuthMode</b> <mode-string><br> +Sets the authentication mode for the currently selected <a href="netstream.html">network stream driver</a>. <mode-string> is driver specifc.</li> +<li><b>$InputTCPServerStreamDriverPermittedPeer</b> <id-string><br> Sets permitted peer IDs. Only these peers are able to connect to the listener. <id-string> semantics depend on the currently selected AuthMode and <a href="netstream.html">network stream driver</a>. PermittedPeers may not be set in anonymous modes.</li> +<li><b>$InputTCPServerBindRuleset</b> <ruleset><br> +Binds the listener to a specific <a href="multi_ruleset.html">ruleset</a>.</li> </ul> <b>Caveats/Known Bugs:</b> <ul> @@ -62,20 +81,22 @@ AuthMode and <a href="netstream.html">network stream driver</a>. Permitted <li>can not be loaded together with <a href="imgssapi.html">imgssapi</a> (which includes the functionality of imtcp)</li> </ul> -<p><b>Sample:</b></p> -<p>This sets up a TCP server on port 514:<br> +<p><b>Example:</b></p> +<p>This sets up a TCP server on port 514 and permits it to accept up to 500 connections:<br> </p> -<textarea rows="15" cols="60">$ModLoad imtcp # -needs to be done just once +<textarea rows="15" cols="60">$ModLoad imtcp # needs to be done just once +$InputTCPMaxSessions 500 $InputTCPServerRun 514 </textarea> +<p>Note that the parameters (here: max sessions) need to be set <b>before</b> the listener +is activated. Otherwise, the parameters will not apply. +</p> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> -<p><font size="2">This documentation is part of the -<a href="http://www.rsyslog.com/">rsyslog</a> +<p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +Copyright © 2008,2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html> +</body> +</html> diff --git a/doc/imudp.html b/doc/imudp.html new file mode 100644 index 00000000..f0e86307 --- /dev/null +++ b/doc/imudp.html @@ -0,0 +1,58 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>TCP Syslog Input Module</title> +</head> + +<body> +<a href="rsyslog_conf_modules.html">back to rsyslog module overview</a> + +<h1>UDP Syslog Input Module</h1> +<p><b>Module Name: imudp</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Multi-Ruleset Support: </b>since 5.3.2 +<p><b>Description</b>:</p> +<p>Provides the ability to receive syslog messages via UDP. +<p>Multiple receivers may be configured by specifying +$UDPServerRun multiple times. +</p> +<p><b>Configuration Directives</b>:</p> +<ul> +<li>$UDPServerAddress <IP><br> +local IP address (or name) the UDP listens should bind to</li> +<li>$UDPServerRun <port><br> +former -r<port> option, default 514, start UDP server on this +port, "*" means all addresses</li> +<li>$UDPServerTimeRequery <nbr-of-times><br> +this is a performance +optimization. Getting the system time is very costly. With this setting, imudp can +be instructed to obtain the precise time only once every n-times. This logic is +only activated if messages come in at a very fast rate, so doing less frequent +time calls should usually be acceptable. The default value is two, because we have +seen that even without optimization the kernel often returns twice the identical time. +You can set this value as high as you like, but do so at your own risk. The higher +the value, the less precise the timestamp. +<li>$InputUDPServerBindRuleset <ruleset><br> +Binds the listener to a specific <a href="multi_ruleset.html">ruleset</a>.</li> +</ul> +<b>Caveats/Known Bugs:</b> +<ul> +<li>currently none known</li> +</ul> +<p><b>Sample:</b></p> +<p>This sets up an UPD server on port 514:<br> +</p> +<textarea rows="15" cols="60">$ModLoad imudp # needs to be done just once +$UDPServerRun 514 +</textarea> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer +Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 472470a0..15c365a6 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -46,6 +46,18 @@ Ignore timestamps included in the messages, applies to messages received via the <li><b>$SystemLogSocketName</b> <name-of-socket> -- former -p option</li> <li><b>$SystemLogFlowControl</b> [on/<b>off</b>] - specifies if flow control should be applied to the system log socket.</li> +<li><b>$InputUnixListenSocketCreatePath</b> [on/<b>off</b>] - create directories in the socket path +if they do not already exist. They are created with 0755 permissions with the owner being the process under +which rsyslogd runs. The default is not to create directories. Keep in mind, though, that rsyslogd always +creates the socket itself if it does not exist (just not the directories by default). +<br>Note that this statement affects the +next $AddUnixListenSocket directive that follows in sequence in the configuration file. It never works +on the system log socket (where it is deemed unnecessary). Also note that it is automatically +being reset to "off" after the $AddUnixListenSocket directive, so if you would have it active +for two additional listen sockets, you need to specify it in front of each one. This option is primarily considered +useful for defining additional sockets that reside on non-permanent file systems. As rsyslogd probably starts +up before the daemons that create these sockets, it is a vehicle to enable rsyslogd to listen to those +sockets even though their directories do not yet exist. [available since 4.7.0 and 5.3.0]</li> <li><b>$AddUnixListenSocket</b> <name-of-socket> adds additional unix socket, default none -- former -a option</li> <li><b>$InputUnixListenSocketHostName</b> <hostname> permits to override the hostname that shall be used inside messages taken from the <b>next</b> $AddUnixListenSocket socket. Note that @@ -57,20 +69,32 @@ that the local hostname can be overridden in cases where that is desired.</li> <br> This documentation is sparse and incomplete. <p><b>Sample:</b></p> -<p>The following sample is the minimum setup required to accept syslog messages from applications running on the local system.<br> +<p>The following sample is the minimum setup required to accept syslog messages from applications running +on the local system.<br> </p> <textarea rows="2" cols="70">$ModLoad imuxsock # needs to be done just once $SystemLogSocketFlowControl on # enable flow control (use if needed) </textarea> <p>The following sample is a configuration where rsyslogd pulls logs from two jails, and assigns different hostnames to each of the jails: </p> -<textarea rows="6" cols="60">$ModLoad imuxsock # needs to be done just once +<textarea rows="6" cols="70">$ModLoad imuxsock # needs to be done just once $InputUnixListenSocketHostName jail1.example.net $AddUnixListenSocket /jail/1/dev/log $InputUnixListenSocketHostName jail2.example.net $AddUnixListenSocket /jail/2/dev/log </textarea> +<p>The following sample is a configuration where rsyslogd reads the openssh log +messages via a separate socket, but this socket is created on a temporary file +system. As rsyslogd starts up before the sshd, it needs to create the socket +directories, because it otherwise can not open the socket and thus not listen +to openssh messages. Note that it is vital not to place any other socket between +the $InputUnixListenSocketCreatePath and the $InputUnixListenSocketHostName.</p> +<textarea rows="6" cols="70">$ModLoad imuxsock # needs to be done just once + +$InputUnixListenSocketCreatePath on # turn on for *next* socket +$InputUnixListenSocketHostName /var/run/sshd/dev/log +</textarea> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the diff --git a/doc/index.html b/doc/index.html index 349c8e57..b3b336a7 100644 --- a/doc/index.html +++ b/doc/index.html @@ -25,8 +25,7 @@ To do the really cool things, though, you need to learn a bit about its new features. The man pages offer a bare minimum of information (and are still quite long). Read the <a href="manual.html">html documentation</a> instead. -When you change the configuration, -remember to restart (or HUP) rsyslogd, because otherwise it won't use your -new settings (and you'll end up totally puzzled why this great config of yours -does not even work a bit...;)) +When you change the configuration, remember to restart rsyslogd, because otherwise +it will not use your new settings (and you'll end up totally puzzled why this great +config of yours does not even work a bit...;)) </body></html> diff --git a/doc/man_rsyslogd.html b/doc/man_rsyslogd.html deleted file mode 100644 index d18fd88a..00000000 --- a/doc/man_rsyslogd.html +++ /dev/null @@ -1,438 +0,0 @@ -<BODY><PRE> -RSYSLOGD(8) Linux System Administration RSYSLOGD(8) - - - -<B>NAME</B> - rsyslogd - reliable and extended syslogd - -<B>SYNOPSIS</B> - <B>rsyslogd </B>[ <B>-4 </B>] [ <B>-6 </B>] [ <B>-A </B>] [ <B>-a </B><I>socket </I>] [ <B>-d </B>] [ <B>-e </B>] - [ <B>-f </B><I>config file </I>] [ <B>-h </B>] [ <B>-i </B><I>pid file </I>] [ <B>-l </B><I>hostlist </I>] - [ <B>-m </B><I>interval </I>] [ <B>-n </B>] [ <B>-o </B>] [ <B>-p </B><I>socket </I>] - [ <B>-r </B><I>[port] </I>] [ <B>-s </B><I>domainlist </I>] [ <B>-t </B><I>port,max-nbr-of-sessions </I>] - [ <B>-v </B>] [ <B>-w </B>] [ <B>-x </B>] - - -<B>DESCRIPTION</B> - <B>Rsyslogd </B>is a system utility providing support for message logging. - Support of both internet and unix domain sockets enables this utility - to support both local and remote logging (via UDP and TCP). - - <B>Rsyslogd</B>(8) is derived from the sysklogd package which in turn is - derived from the stock BSD sources. - - <B>Rsyslogd </B>provides a kind of logging that many modern programs use. - Every logged message contains at least a time and a hostname field, - normally a program name field, too, but that depends on how trusty the - logging program is. The rsyslog package supports free definition of - output formats via templates. It also supports precise timestamps and - writing directly to MySQL databases. If the database option is used, - tools like phpLogCon can be used to view the log data. - - While the <B>rsyslogd </B>sources have been heavily modified a couple of notes - are in order. First of all there has been a systematic attempt to - insure that rsyslogd follows its default, standard BSD behavior. Of - course, some configuration file changes are necessary in order to sup- - port the template system. However, rsyslogd should be able to use a - standard syslog.conf and act like the original syslogd. However, an - original syslogd will not work correctly with a rsyslog-enhanced con- - figuration file. At best, it will generate funny looking file names. - The second important concept to note is that this version of rsyslogd - interacts transparently with the version of syslog found in the stan- - dard libraries. If a binary linked to the standard shared libraries - fails to function correctly we would like an example of the anomalous - behavior. - - The main configuration file <I>/etc/rsyslog.conf </I>or an alternative file, - given with the <B>-f </B>option, is read at startup. Any lines that begin - with the hash mark (‘‘#’’) and empty lines are ignored. If an error - occurs during parsing the error element is ignored. It is tried to - parse the rest of the line. - - For details and configuration examples, see the <B>rsyslog.conf (5) </B>man - page. - - - -<B>OPTIONS</B> - <B>-A </B>When sending UDP messages, there are potentially multiple paths - to the target destination. By default, <B>rsyslogd </B>only sends to - the first target it can successfully send to. If -A is given, - messages are sent to all targets. This may improve reliability, - but may also cause message duplication. This option should - enabled only if it is fully understood. - - <B>-4 </B>Causes <B>rsyslogd </B>to listen to IPv4 addresses only. If neither -4 - nor -6 is given, <B>rsyslogd </B>listens to all configured addresses of - the system. - - <B>-6 </B>Causes <B>rsyslogd </B>to listen to IPv6 addresses only. If neither -4 - nor -6 is given, <B>rsyslogd </B>listens to all configured addresses of - the system. - - <B>-a </B><I>socket</I> - Using this argument you can specify additional sockets from that - <B>rsyslogd </B>has to listen to. This is needed if you’re going to - let some daemon run within a chroot() environment. You can use - up to 19 additional sockets. If your environment needs even - more, you have to increase the symbol <B>MAXFUNIX </B>within the sys- - logd.c source file. An example for a chroot() daemon is - described by the people from OpenBSD at - http://www.psionic.com/papers/dns.html. - - <B>-d </B>Turns on debug mode. Using this the daemon will not proceed a - <B>fork</B>(2) to set itself in the background, but opposite to that - stay in the foreground and write much debug information on the - current tty. See the DEBUGGING section for more information. - - <B>-e </B>Set the default of $RepeatedMsgReduction config option to "off". - Hine: "e" like "every message". For further information, see - there. - - <B>-f </B><I>config file</I> - Specify an alternative configuration file instead of <I>/etc/rsys-</I> - <I>log.conf</I>, which is the default. - - <B>-h </B>By default rsyslogd will not forward messages it receives from - remote hosts. Specifying this switch on the command line will - cause the log daemon to forward any remote messages it receives - to forwarding hosts which have been defined. - - <B>-i </B><I>pid file</I> - Specify an alternative pid file instead of the default one. - This option must be used if multiple instances of rsyslogd - should run on a single machine. - - <B>-l </B><I>hostlist</I> - Specify a hostname that should be logged only with its simple - hostname and not the fqdn. Multiple hosts may be specified - using the colon (‘‘:’’) separator. - - <B>-m </B><I>interval</I> - The <B>rsyslogd </B>logs a mark timestamp regularly. The default - <I>interval </I>between two <I>-- MARK -- </I>lines is 20 minutes. This can - be changed with this option. Setting the <I>interval </I>to zero turns - it off entirely. - - <B>-n </B>Avoid auto-backgrounding. This is needed especially if the - <B>rsyslogd </B>is started and controlled by <B>init</B>(8). - - <B>-o </B>Omit reading the standard local log socket. This option is most - useful for running multiple instances of rsyslogd on a single - machine. When specified, no local log socket is opened at all. - - <B>-p </B><I>socket</I> - You can specify an alternative unix domain socket instead of - <I>/dev/log</I>. - - <B>-r </B><I>["port"]</I> - Activates the syslog/udp listener service. The listener will - listen to the specified port. If no port is specified, 0 is - used as port number, which in turn will lead to a lookup of the - system default syslog port. If there is no system default, 514 - is used. Please note that the port must immediately follow the - -r option. Thus "-r514" is valid while "-r 514" is invalid (note - the space). - - <B>-s </B><I>domainlist</I> - Specify a domainname that should be stripped off before logging. - Multiple domains may be specified using the colon (‘‘:’’) sepa- - rator. Please be advised that no sub-domains may be specified - but only entire domains. For example if <B>-s north.de </B>is speci- - fied and the host logging resolves to satu.infodrom.north.de no - domain would be cut, you will have to specify two domains like: - <B>-s north.de:infodrom.north.de</B>. - - <B>-t </B><I>port,max-nbr-of-sessions</I> - Activates the syslog/tcp listener service. The listener will - listen to the specified port. If max-nbr-of-sessions is speci- - fied, that becomes the maximum number of concurrent tcp ses- - sions. If not specified, the default is 200. Please note that - syslog/tcp is not standardized, but the implementation in rsys- - logd follows common practice and is compatible with e.g. Cisco - PIX, syslog-ng and MonitorWare (Windows). Please note that the - port must immediately follow the -t option. Thus "-t514" is - valid while "-t 514" is invalid (note the space). - - <B>-v </B>Print version and exit. - - <B>-w </B>Supress warnings issued when messages are received from non- - authorized machines (those, that are in no AllowedSender list). - - <B>-x </B>Disable DNS for remote messages. - - -<B>SIGNALS</B> - <B>Rsyslogd </B>reacts to a set of signals. You may easily send a signal to - <B>rsyslogd </B>using the following: - - kill -SIGNAL ‘cat /var/run/rsyslogd.pid‘ - - - <B>SIGHUP </B>This lets <B>rsyslogd </B>perform a re-initialization. All open files - are closed, the configuration file (default is <I>/etc/rsys-</I> - <I>log.conf</I>) will be reread and the <B>rsyslog</B>(3) facility is started - again. - - <B>SIGTERM</B> - <B>Rsyslogd </B>will die. - - <B>SIGINT</B>, <B>SIGQUIT</B> - If debugging is enabled these are ignored, otherwise <B>rsyslogd</B> - will die. - - <B>SIGUSR1</B> - Switch debugging on/off. This option can only be used if <B>rsys-</B> - <B>logd </B>is started with the <B>-d </B>debug option. - - <B>SIGCHLD</B> - Wait for childs if some were born, because of wall’ing messages. - - -<B>SUPPORT FOR REMOTE LOGGING</B> - <B>Rsyslogd </B>provides network support to the syslogd facility. Network - support means that messages can be forwarded from one node running - rsyslogd to another node running rsyslogd (or a compatible syslog - implementation) where they will be actually logged to a disk file. - - To enable this you have to specify either the <B>-r </B>or <B>-t </B>option on the - command line. The default behavior is that <B>rsyslogd </B>won’t listen to - the network. You can also combine these two options if you want rsys- - logd to listen to both TCP and UDP messages. - - The strategy is to have rsyslogd listen on a unix domain socket for - locally generated log messages. This behavior will allow rsyslogd to - inter-operate with the syslog found in the standard C library. At the - same time rsyslogd listens on the standard syslog port for messages - forwarded from other hosts. To have this work correctly the <B>ser-</B> - <B>vices</B>(5) files (typically found in <I>/etc</I>) must have the following entry: - - syslog 514/udp - - If this entry is missing <B>rsyslogd </B>will use the well known port of 514 - (so in most cases, it’s not really needed). - - To cause messages to be forwarded to another host replace the normal - file line in the <I>rsyslog.conf </I>file with the name of the host to which - the messages is to be sent prepended with an @ (for UDP delivery) or - the sequence @@ (for TCP delivery). The host name can also be followed - by a colon and a port number, in which case the message is sent to the - specified port on the remote host. - - For example, to forward <B>ALL </B>messages to a remote host use the - following <I>rsyslog.conf </I>entry: - - # Sample rsyslogd configuration file to - # messages to a remote host forward all. - *.* @hostname - More samples can be found in sample.conf. - - If the remote hostname cannot be resolved at startup, because - the name-server might not be accessible (it may be started after - rsyslogd) you don’t have to worry. <B>Rsyslogd </B>will retry to - resolve the name ten times and then complain. Another possibil- - ity to avoid this is to place the hostname in <I>/etc/hosts</I>. - - With normal <B>syslogd</B>s you would get syslog-loops if you send out - messages that were received from a remote host to the same host - (or more complicated to a third host that sends it back to the - first one, and so on). - - To avoid this no messages that were received from a remote host - are sent out to another (or the same) remote host. You can dis- - able this feature by the <B>-h </B>option. - - If the remote host is located in the same domain as the host, - <B>rsyslogd </B>is running on, only the simple hostname will be logged - instead of the whole fqdn. - - In a local network you may provide a central log server to have - all the important information kept on one machine. If the net- - work consists of different domains you don’t have to complain - about logging fully qualified names instead of simple hostnames. - You may want to use the strip-domain feature <B>-s </B>of this server. - You can tell <B>rsyslogd </B>to strip off several domains other than - the one the server is located in and only log simple hostnames. - - Using the <B>-l </B>option there’s also a possibility to define single - hosts as local machines. This, too, results in logging only - their simple hostnames and not the fqdns. - - -<B>OUTPUT TO DATABASES</B> - <B>Rsyslogd </B>has support for writing data to MySQL database tables. The - exact specifics are described in the <B>rsyslog.conf (5) </B>man page. Be sure - to read it if you plan to use database logging. - - While it is often handy to have the data in a database, you must be - aware of the implications. Most importantly, database logging takes far - longer than logging to a text file. A system that can handle a large - log volume when writing to text files can most likely not handle a sim- - ilar large volume when writing to a database table. - - -<B>OUTPUT TO NAMED PIPES (FIFOs)</B> - <B>Rsyslogd </B>has support for logging output to named pipes (fifos). A fifo - or named pipe can be used as a destination for log messages by prepend- - ing a pipy symbol (‘‘|’’) to the name of the file. This is handy for - debugging. Note that the fifo must be created with the mkfifo command - before <B>rsyslogd </B>is started. - - The following configuration file routes debug messages from the - kernel to a fifo: - - # Sample configuration to route kernel debugging - # messages ONLY to /usr/adm/debug which is a - # named pipe. - kern.=debug |/usr/adm/debug - - -<B>INSTALLATION CONCERNS</B> - There is probably one important consideration when installing rsyslogd. - It is dependent on proper formatting of messages by the syslog func- - tion. The functioning of the syslog function in the shared libraries - changed somewhere in the region of libc.so.4.[2-4].n. The specific - change was to null-terminate the message before transmitting it to the - <I>/dev/log </I>socket. Proper functioning of this version of rsyslogd is - dependent on null-termination of the message. - - This problem will typically manifest itself if old statically linked - binaries are being used on the system. Binaries using old versions of - the syslog function will cause empty lines to be logged followed by the - message with the first character in the message removed. Relinking - these binaries to newer versions of the shared libraries will correct - this problem. - - The <B>rsyslogd</B>(8) can be run from <B>init</B>(8) or started as part of the rc.* - sequence. If it is started from init the option <I>-n </I>must be set, other- - wise you’ll get tons of syslog daemons started. This is because - <B>init</B>(8) depends on the process ID. - - -<B>SECURITY THREATS</B> - There is the potential for the rsyslogd daemon to be used as a conduit - for a denial of service attack. A rogue program(mer) could very easily - flood the rsyslogd daemon with syslog messages resulting in the log - files consuming all the remaining space on the filesystem. Activating - logging over the inet domain sockets will of course expose a system to - risks outside of programs or individuals on the local machine. - - There are a number of methods of protecting a machine: - - 1. Implement kernel firewalling to limit which hosts or networks - have access to the 514/UDP socket. - - 2. Logging can be directed to an isolated or non-root filesystem - which, if filled, will not impair the machine. - - 3. The ext2 filesystem can be used which can be configured to limit - a certain percentage of a filesystem to usage by root only. - <B>NOTE </B>that this will require rsyslogd to be run as a non-root - process. <B>ALSO NOTE </B>that this will prevent usage of remote log- - ging since rsyslogd will be unable to bind to the 514/UDP - socket. - - 4. Disabling inet domain sockets will limit risk to the local - machine. - - 5. Use step 4 and if the problem persists and is not secondary to a - rogue program/daemon get a 3.5 ft (approx. 1 meter) length of - sucker rod* and have a chat with the user in question. - - Sucker rod def. — 3/4, 7/8 or 1in. hardened steel rod, male - threaded on each end. Primary use in the oil industry in West- - ern North Dakota and other locations to pump ’suck’ oil from oil - wells. Secondary uses are for the construction of cattle feed - lots and for dealing with the occasional recalcitrant or bel- - ligerent individual. - - <B>Message replay and spoofing</B> - If remote logging is enabled, messages can easily be spoofed and - replayed. As the messages are transmitted in clear-text, an attacker - might use the information obtained from the packets for malicious - things. Also, an attacker might reply recorded messages or spoof a - sender’s IP address, which could lead to a wrong perception of system - activity. Be sure to think about syslog network security before - enabling it. - - -<B>DEBUGGING</B> - When debugging is turned on using <B>-d </B>option then <B>rsyslogd </B>will be very - verbose by writing much of what it does on stdout. Whenever the con- - figuration file is reread and re-parsed you’ll see a tabular, corre- - sponding to the internal data structure. This tabular consists of four - fields: - - <I>number </I>This field contains a serial number starting by zero. This num- - ber represents the position in the internal data structure (i.e. - the array). If one number is left out then there might be an - error in the corresponding line in <I>/etc/rsyslog.conf</I>. - - <I>pattern</I> - This field is tricky and represents the internal structure - exactly. Every column stands for a facility (refer to <B>sys-</B> - <B>log</B>(3)). As you can see, there are still some facilities left - free for former use, only the left most are used. Every field - in a column represents the priorities (refer to <B>syslog</B>(3)). - - <I>action </I>This field describes the particular action that takes place - whenever a message is received that matches the pattern. Refer - to the <B>syslog.conf</B>(5) manpage for all possible actions. - - <I>arguments</I> - This field shows additional arguments to the actions in the last - field. For file-logging this is the filename for the logfile; - for user-logging this is a list of users; for remote logging - this is the hostname of the machine to log to; for console-log- - ging this is the used console; for tty-logging this is the spec- - ified tty; wall has no additional arguments. - - - <B>templates</B> - There will also be a second internal structure which lists all - defined templates and there contents. This also enables you to - see the internally-defined, hardcoded templates. - -<B>FILES</B> - <I>/etc/rsyslog.conf</I> - Configuration file for <B>rsyslogd</B>. See <B>rsyslog.conf</B>(5) for exact - information. - <I>/dev/log</I> - The Unix domain socket to from where local syslog messages are - read. - <I>/var/run/rsyslogd.pid</I> - The file containing the process id of <B>rsyslogd</B>. - -<B>BUGS</B> - Please review the file BUGS for up-to-date information on known bugs - and annoyances. - -<B>Further Information</B> - Please visit <B>http://www.rsyslog.com/doc </B>for additional information, - tutorials and a support forum. - -<B>SEE ALSO</B> - <B>rsyslog.conf</B>(5), <B>logger</B>(1), <B>syslog</B>(2), <B>syslog</B>(3), <B>services</B>(5), - <B>savelog</B>(8) - - -<B>COLLABORATORS</B> - <B>rsyslogd </B>is derived from sysklogd sources, which in turn was taken from - the BSD sources. Special thanks to Greg Wettstein (greg@wind.enjel- - lic.com) and Martin Schulze (joey@linux.de) for the fine sysklogd pack- - age. - - Rainer Gerhards - Adiscon GmbH - Grossrinderfeld, Germany - rgerhards@adiscon.com - - Michael Meckelein - Adiscon GmbH - mmeckelein@adiscon.com - - - -Version 1.16.1 (devel) 17 July 2007 RSYSLOGD(8) -</PRE></BODY> diff --git a/doc/manual.html b/doc/manual.html index adf89587..bcdd64cf 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,32 +19,35 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 4.3.1 (devel branch) of rsyslog.</b> -Visit the <i> <a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b> to obtain current -version information and project status. +<p><b>This documentation is for version 5.5.2 (devel branch) of rsyslog.</b> +Visit the <i><a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b> +to obtain current version information and project status. </p><p><b>If you like rsyslog, you might want to lend us a helping hand. </b>It doesn't require a lot of time - even a single mouse click helps. Learn <a href="how2help.html">how to help the rsyslog project</a>. Due to popular demand, there is now a <a href="rsyslog_ng_comparison.html">side-by-side comparison between rsyslog and syslog-ng</a>.</p> <p>If you are upgrading from rsyslog v2 or stock sysklogd, -<a href="v3compatibility.html">be -sure to read the rsyslog v3 compatibility document!</a> It will work even +<a href="v3compatibility.html">be sure to read the rsyslog v3 compatibility notes</a>, +and if you are upgrading from v3, read the +<a href="v4compatibility.html">rsyslog v4 compatibility notes</a> and +if you upgrade from v4, read the +<a href="v5compatibility.html">rsyslog v5 compatibility notes</a>. +<p>Rsyslog will work even if you do not read the doc, but doing so will definitely improve your experience.</p> -<p><span style="font-weight: bold;"></span><b>Follow -the links below for the</b><br></p><ul> - +<p><b>Follow the links below for the</b></p> +<ul> <li><a href="troubleshoot.html">troubleshooting rsyslog problems</a></li> <li><a href="rsyslog_conf.html">configuration file syntax (rsyslog.conf)</a></li> <li><a href="http://www.rsyslog.com/tool-regex">a regular expression checker/generator tool for rsyslog</a></li> <li> <a href="property_replacer.html">property replacer, an important core component</a></li> <li>a commented <a href="sample.conf.html">sample rsyslog.conf</a> </li> <li><a href="bugs.html">rsyslog bug list</a></li> -<li><a href="rsyslog_packages.html"> rsyslog packages</a></li> -<li><a href="generic_design.html">backgrounder on -generic syslog application design</a><!-- not good as it currently is ;) <li><a href="contributors.html">contributor "Hall of Fame"</a>--></li> +<li><a href="messageparser.html">understanding rsyslog message parsers</a></li> +<li><a href="generic_design.html">backgrounder on generic syslog application design</a></li> <li><a href="modules.html">description of rsyslog modules</a></li> -<li><a href="man_rsyslogd.html">rsyslogd man page</a> (heavily outdated)</li> +<li><a href="rsyslog_packages.html">rsyslog packages</a></li> +<li><a href="http://cookbook.rsyslog.com">the rsyslog "cookbook"</a> - a set of configurations ready to use</li> </ul> <p><b>We have some in-depth papers on</b></p> <ul> @@ -52,8 +55,10 @@ generic syslog application design</a><!-- not good as it currently is ;) <li><a <li><a href="build_from_repo.html">obtaining rsyslog from the source repository</a></li> <li><a href="ipv6.html">rsyslog and IPv6</a> (which is fully supported)</li> <li><a href="rsyslog_secure_tls.html">native TLS encryption for syslog</a></li> +<li><a href="multi_ruleset.html">using multiple rule sets in rsyslog</a></li> <li><a href="rsyslog_stunnel.html">ssl-encrypting syslog with stunnel</a></li> <li><a href="rsyslog_mysql.html">writing syslog messages to MySQL (and other databases as well)</a></li> +<li><a href="rsyslog_pgsql.html">writing syslog messages to PostgreSQL (and other databases as well)</a></li> <li><a href="rsyslog_high_database_rate.html">writing massive amounts of syslog messages to a database</a></li> <li><a href="rsyslog_reliable_forwarding.html">reliable forwarding to a remote server</a></li> <li><a href="rsyslog_php_syslog_ng.html">using @@ -66,6 +71,7 @@ syslog sender over NAT</a> (online only)</li> <li><a href="debug.html">debug support in rsyslog</a></li> <li>Developer Documentation <ul> + <li><a href="build_from_repo.html">building rsyslog from the source repository</a></li> <li><a href="dev_oplugins.html">writing rsyslog output plugins</a></li> <li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> </ul></li> diff --git a/doc/messageparser.html b/doc/messageparser.html new file mode 100644 index 00000000..370db59f --- /dev/null +++ b/doc/messageparser.html @@ -0,0 +1,222 @@ +<html> +<head> +<title>Message parsers in rsyslog</title> +</head> +<body> +<a href="manual.html">rsyslog documentation</a> + +<h1>Message parsers in rsyslog</h1> +<p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> +(2009-11-06)</i></small></p> +<h2>Intro</h2> +<p>Message parsers are a feature of rsyslog 5.3.4 and above. In this article, I describe what +message parsers are, what they can do and how they relate to the relevant standards. I will +also describe what you can not do with time. Finally, I give some advice on implementing your +own custom parser. + +<h2>What are message parsers?</h2> +<p>Well, the quick answer is that message parsers are the component of rsyslog that +parses the syslog message after it is being received. Prior to rsyslog 5.3.4, message parsers +where built in into the rsyslog core itself and could not be modified (other than by modifying +the rsyslog code). +<p>In 5.3.4, we changed that: message parsers are now loadable modules (just +like input and output modules). That means that new message parsers can be added without +modifying the rsyslog core, even without contributing something back to the +project. +<p>But that doesn't answer what a message parser really is. What does ist mean to "parse a +message" and, maybe more importantly, what is a message? To answer these questions correctly, +we need to dig down into the relevant standards. +<a href="http://tools.ietf.org/html/rfc5424">RFC5424</a> specifies a layered architecture +for the syslog protocol: +<p align="center"><img src="rfc5424layers.png" alt="RFC5424 syslog protocol layers"> +<p>For us important is the distinction between the syslog transport and the upper layers. +The transport layer specifies how a stream of messages is assembled at the sender side and +how this stream of messages is disassembled into the individual messages at the receiver +side. In networking terminology, this is called "framing". The core idea is that +each message is put into a so-called "frame", which then is transmitted over the communications +link. +<p>The framing used is depending on the protocol. For example, in UDP the "frame"-equivalent is +a packet that is being sent (this also means that no two messages can travel within a single +UDP packet). In "plain tcp syslog", the industry standard, LF is used as a frame delimiter +(which also means that no multi-line message can properly be transmitted, a "design" flaw +in plain tcp syslog). In <a href="http://tools.ietf.org/html/rfc5425">RFC5425</a> there is +a header in front of each frame that contains the size of the message. With this framing, +any message content can properly be transferred. +<p>And now comes the important part: <b>message parsers do NOT operate at the transport +layer</b>, they operate, as their name implies, on messages. So we can not use message +parsers to change the underlying framing. For example, if a sender splits (for whatever +reason) a single message into two and encapsulates these into two frames, there is no way +a message parser could undo that. +<p>A typical example may be a multi-line message: let's assume some originator has generated +a message for the format "A\nB" (where \n means LF). If that message is being transmitted +via plain tcp syslog, the frame delimiter is LF. So the sender will delimite the frame with +LF, but otherwise send the message unmodified onto the wire (because that is how things are +-unfortunately- done in plain tcp syslog...). So wire will see "A\nB\n". When this +arrives at the receiver, the transport layer will undo the framing. When it sees the LF +after A, it thinks it finds a valid frame delimiter (in fact, this is the correct view!). So +the receive will extract one complete message A and one complete message B, not knowing +that they once were both part of a large multi-line message. These two messages are then +passed to the upper layers, where the message parsers receive them and extract information. +However, the message parsers never know (or even have a chance to see) that A and B +belonged together. Even further, in rsyslog there is no guarnatee that A will be parsed +before B - concurrent operations may cause the reverse order (and do so very validly). +<p>The important lesson is: <b>message parsers can not be used to fix a broken framing</b>. +You need a full protocol implementation to do that, what is the domain of input and +output modules. +<p>I have now told you what you can not do with message parsers. But what they are good for? +Thankfully, broken framing is not the primary problem of the syslog world. A wealth of different +formats is. Unfortunately, many real-world implementations violate the relevant standards +in one way or another. That makes it often very hard to extract meaningful information from +a message or to process messages from different sources by the same rules. In my article +<a href="syslog_parsing.html">syslog parsing in rsyslog</a> I have elaborated on all +the real-world evil that you can usually see. So I won't repeat that here. But in short, the +real problem is not the framing, but how to make malformed messages well-looking. +<p><b>This is what message parsers permit you to do: take a (well-known) malformed message, parse +it according to its semantics and generate perfectly valid internal message representations +from it.</b> So as long as messages are consistenly in the same wrong format (and they usually +are!), a message parser can look at that format, parse it, and make the message processable just +like it were wellformed in the first place. Plus, one can abuse the interface to do some other +"intersting" tricks, but that would take us to far. +<p>While this functionality may not sound exciting, it actually solves a very big issue (that you +only really understand if you have managed a system with various different syslog sources). +Note that we were often able to process malformed messages in the past with the help of the +property replacer and regular expressions. While this is nice, it has a performance hit. A +message parser is a C code, compiled to native language, and thus typically much faster than +any regular expression based method (depending, of course, on the quality of the implementation...). + +<h2>How are message parsers used?</h2> +<p>In a simlified view, rsyslog +<ol> +<li>first receives messages (via the input module), +<li><i>then parses them (at the message level!)</i> and +<li>then processes them (operating on the internal message representation). +</ol> +Message parsers are utilized in the second step (written in italics). +Thus, they take the raw message (NOT frame!) received from the remote system and create +the internal structure out of it that the other parts of rsyslog need in order to perform +their processing. Parsing is vital, because an unparsed message can not be processed in the +third stage, the actual application-level processing (like forwarding or writing to files). +<h3>Parser Chains and how they Operate</h3> +Rsyslog chains parsers together to provide flexibility. +A <b>parser chain</b> +contains all parsers that can potentially be used to parse a message. +It is assumed that there is some +way a parser can detect if the message it is being presented is supported by it. If so, the parser +will tell the rsyslog engine and parse the message. The rsyslog engine now calls each parser +inside the chain (in sequence!) until the first parser is able to parse the message. After one +parser has been found, the message is considered parsed and no others parsers are called on that +message. +<p>Side-note: this method implies there are some "not-so-dirty" tricks available to modify +the message by a parser module that declares itself as "unable to parse" but still does +some message modification. This was not a primary design goal, but may be utilized, and the +interface probably extended, to support generic filter modules. These would need to go +to the root of the parser chain. As mentioned, the current system already supports this. +<p>The position inside the parser chain can be thought of as a priority: parser sitting +earlier in the chain take precedence over those sitting later in it. So more specific +parser should go ealier in the chain. A good example of how this works is the default parser +set provided by rsyslog: rsyslog.rfc5424 and rsyslog.rfc3164, each one parses according to the +rfc that has named it. RFC5424 was designed to be distinguishable from RFC3164 message by the +sequence "1 " immediately after the so-called PRI-part (don't worry about these words, it is +sufficient if you understand there is a well-defined sequence used to indentify RFC5424 +messages). In contrary, RFC3164 actually permits everything as a valid message. Thus the +RFC3164 parser will always parse a message, sometimes with quite unexpected outcome (there is +a lot of guesswork involved in that parser, which unfortunately is unavoidable due to +existing techology limits). So the default parser chain is to try the RFC5424 parser first +and after it the RFC3164 parser. If we have a 5424-formatted message, that parser will +identify and parse it and the rsyslog engine will stop processing. But if we receive a +legacy syslog message, the RFC5424 will detect that it can not parse it, return this status +to the engine which then calls the next parser inside the chain. That usually happens to be +the RFC3164 parser, which will always process the message. But there could also be any other +parser inside the chain, and then each one would be called unless one that is able to parse +can be found. +<p>If we reversed the parser order, RFC5424 messages would incorrectly parsed. Why? Because the +RFC3164 parser will always parse every message, so if it were asked first, it would parse +(and misinterpret) the 5424-formatted message, return it did so and the rsyslog engine would +never call the 5424 parser. So oder of sequence is very important. +<p>What happens if no parser in the chain could parse a message? Well, then we could not +obtain the in-memory representation that is needed to further process the message. In that +case, rsyslog has no other choice than to discard the message. If it does so, it will emit +a warning message, but only in the first 1,000 incidents. This limit is a safety measure +against message-loops, which otherwise could quickly result from a parser chain +misconfiguration. <b>If you do not tolerate loss of unparsable messages, you must ensure +that each message can be parsed.</b> You can easily achive this by always using the +"rsyslog-rfc3164" parser as the <i>last</i> parser inside parser chains. That may result +in invalid parsing, but you will have a chance to see the invalid message (in debug mode, +a warning message will be written to the debug log each time a message is dropped due to +inability to parse it). +<h3>Where are parser chains used?</h3> +<p>We now know what parser chains are and how they operate. The question is now how many +parser chains can be active and how it is decicded which parser chain is used on which message. +This is controlled via <a href="multi_ruleset.html">rsyslog's rulesets</a>. In short, multiple +rulesets can be defined and there always exist at least one ruleset (for specifcs, follow +the <a href="multi_ruleset.html">link</a>). A parser chain is bound to a specific ruleset. +This is done by virtue of defining parsers via the +<a href="rsconf1_rulesetparser.html">$RulesetParser</a> configuration directive (for specifics, +see there). If no such directive is specified, the default parser chain is used. As of this +writing, the default parser chain always consists of "rsyslog.rfc5424", "rsyslog.rfc3164", in +that order. As soon as a parser is configured, the default list is cleared and the new parser +is added to the end of the (initially empty) ruleset's parser chain. +<p>The important point to know is that parser chains are defined on a per-ruleset basis. +<h3>Can I use different parser chains for different devices?</h3> +<p>The correct answer is: generally yes, but it depends. First of all, remember that input +modules (and specific listeners) may be bound to specific rulesets. As parser chains "reside" +in rulesets, binding to a ruleset also binds to the parser chain that is bound to that ruleset. +As a number one prequisite, the input module must support binding to different rulesets. Not +all do, but their number is growing. For example, the important +<a href="imudp.html">imudp</a> and <a href="imtcp.html">imtcp</a> input modules support +that functionality. Those that do not (for example <a href="im3195">im3195</a>) can only +utilize the default ruleset and thus the parser chain defined in that ruleset. +<p>If you do not know if the input module in question supports ruleset binding, check +its documentation page. Those that support it have the requiered directives. +<p>Note that it is currently under evaluation if rsyslog will support binding parser chains +to specific inputs directly, without depending on the ruleset. There are some concerns that +this may not be necessary but adds considerable complexity to the configuration. So this may +or may not be possible in the future. In any case, if we decide to add it, input modules +need to support it, so this functionality would require some time to implement. +<p>The coockbook recipe for using different parsers for different devices is given +as an actual in-depth example in the <a href="rscon1_rulesetsparser.html">$RulesetParser</a> +configuration directive doc page. In short, it is acomplished by defining specific rulesets +for the required parser chains, definining different listener ports for each of the devices +with different format and binding these listeners to the correct ruleset (and thus parser +chains). Using that approach, a variety of different message formats can be supported +via a single rsyslog instance. + +<h2>Which message parsers are available</h2> +<p>As of this writing, there exist only two message parsers, one for RFC5424 format and one for +legacy syslog (loosely described in +<a href="http://tools.ietf.org/html/rfc3164">RFC3164</a>). These parsers are built-in and +must not be explicitely loaded. However, message parsers can be added with relative ease +by anyone knowing to code in C. Then, they can be loaded via $ModLoad just like any +other loadable module. It is expected that the rsyslog project will be contributed additional +message parsers over time, so that at some point there hopefully is a rich choice of them +(I intend to add a browsable repository as soon as new parsers pop up). +<h3>How to write a message parser?</h3> +<p>As a prequisite, you need to know the exact format that the device is sending. Then, you need +moderate C coding skills, and a little bit of rsyslog internals. I guess the rsyslog specific part +should not be that hard, as almost all information can be gained from the existing parsers. They +are rather simple in structure and can be found under the "./tools" directory. They are named +pmrfc3164.c and pmrfc5424.c. You need to follow the usual loadable module guidelines. +It is my expectation that writing a parser should typically not take longer than a single +day, with maybe a day more to get aquainted with rsyslog. Of course, I am not sure if the number +is actually right. +<p>If you can not program or have no time to do it, Adiscon can also write a message parser +for you as +part of the <a href="http://www.rsyslog/professional-services">rsyslog professional services +offering</a>. +<h2>Conclusion</h2> +<p>Malformed syslog messages are a pain and unfortunately often seen in practice. Message parsers +provide a fast and efficient solution for this problem. Different parsers can be defined for +different devices, and they all convert message information into rsyslog's well-defined +internal format. Message parsers were first introduced in rsyslog 5.3.4 and also offer +some interesting ideas that may be explored in the future - up to full message normalization +capabilities. It is strongly recommended that anyone with a heterogenous environment take +a look at message parser capabilities. + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/module_workflow.png b/doc/module_workflow.png Binary files differnew file mode 100644 index 00000000..e1a72e96 --- /dev/null +++ b/doc/module_workflow.png diff --git a/doc/multi_ruleset.html b/doc/multi_ruleset.html new file mode 100644 index 00000000..da65b4ba --- /dev/null +++ b/doc/multi_ruleset.html @@ -0,0 +1,302 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<title>Multiple Rulesets in rsyslog</title></head> +<body> +<h1>Multiple Rulesets in rsyslog</h1> +<p>Starting with version 4.5.0 and 5.1.1, <a href="http://www.rsyslog.com">rsyslog</a> supports +multiple rulesets within a single configuration. +This is especially useful for routing the recpetion of remote messages to a set of specific rules. +Note that the input module must support binding to non-standard rulesets, so the functionality +may not be available with all inputs. +<p>In this document, I am using <a href="imtcp.html">imtcp</a>, an input module +that supports binding to non-standard rulesets since rsyslog started to support them. +<h2>What is a Ruleset?</h2> +If you have worked with (r)syslog.conf, you know that it is made up of what I call rules (others +tend to call them selectors, a sysklogd term). Each rule consist of a filter and one or more +actions to be carried out when the filter evaluates to true. A filter may be as simple as a +traditional +syslog priority based filter (like "*.*" or "mail.info" or a as complex as a +script-like expression. Details on that are covered in the config file documentation. After the +filter come action specifiers, and an action is something that does something to a message, e.g. +write it to a file or forward it to a remote logging server. + +<p>A traditional configuration file is made up of one or more of these rules. When a new +message arrives, its processing starts with the first rule (in order of appearance in +rsyslog.conf) and continues for each rule until either all rules have been processed or +a so-called "discard" action happens, in which case processing stops and the +message is thrown away (what also happens after the last rule has been processed). + +<p>The <b>multi-ruleset</b> support now permits to specify more than one such rule sequence. +You can think of a traditional config file just as a single default rule set, which is +automatically bound to each of the inputs. This is even what actually happens. When +rsyslog.conf is processed, the config file parser looks for the directive + +<pre>$RuleSet <name> +</pre> + +<p>Where name is any name the user likes (but must not start with "RSYSLOG_", which +is the name space reserved for rsyslog use). If it finds this directive, it begins a new +rule set (if the name was not yet know) or switches to an already-existing one (if the name +was known). All rules defined between this $RuleSet directive and the next one are appended +to the named ruleset. Note that the reserved name "RSYSLOG_DefaultRuleset" is used to +specify rsyslogd's default ruleset. You can use that name whereever you can use a ruleset name, +including when binding an input to it. + +<p>Inside a ruleset, messages are processed as described above: they start with the first rule +and rules are processed in the order of appearance of the configuration file until either +there are no more rules or the discard action is executed. Note that with multiple rulesets +no longer <b>all</b> rsyslog.conf rules are executed but <b>only</b> those that are +contained within the specific ruleset. + +<p>Inputs must explicitely bind to rulesets. If they don't do, the default ruleset is bound. + +<p>This brings up the next question: + +<h2>What does "To bind to a Ruleset" mean?</h2> +<p>This term is used in the same sense as "to bind an IP address to an interface": +it means that a specific input, or part of an input (like a tcp listener) will use a specific +ruleset to "pass its messages to". So when a new message arrives, it will be processed +via the bound ruleset. Rule from all other rulesets are irrelevant and will never be processed. +<p>This makes multiple rulesets very handy to process local and remote message via +seperate means: bind the respective receivers to different rule sets, and you do not need +to seperate the messages by any other method. + +<p>Binding to rulesets is input-specifc. For imtcp, this is done via the + +<pre>$InputTCPServerBindRuleset <name> +</pre> + +directive. Note that "name" must be the name of a ruleset that is already defined +at the time the bind directive is given. There are many ways to make sure this happens, but +I personally think that it is best to define all rule sets at the top of rsyslog.conf and +define the inputs at the bottom. This kind of reverses the traditional recommended ordering, but +seems to be a really useful and straightforward way of doing things. +<h2>Why are rulesets important for different parser configurations?</h2> +<p>Custom message parsers, used to handle differnet (and potentially otherwise-invalid) +message formats, can be bound to rulesets. So multiple rulesets can be a very useful +way to handle devices sending messages in different malformed formats in a consistent +way. Unfortunately, this is not uncommon in the syslog world. An in-depth explanation +with configuration sample can be found at the +<a href="rsconf1_rulesetparser.html">$RulesetParser</a> configuration directive. +<h2>Can I use a different Ruleset as the default?</h2> +<p>This is possible by using the + +<pre>$DefaultRuleset <name> +</pre> + +Directive. Please note, however, that this directive is actually global: that is, it does not +modify the ruleset to which the next input is bound but rather provides a system-wide +default rule set for those inputs that did not explicitly bind to one. As such, the directive +can not be used as a work-around to bind inputs to non-default rulesets that do not support +ruleset binding. +<h2>Examples</h2> +<h3>Split local and remote logging</h3> +<p>Let's say you have a pretty standard system that logs its local messages to the usual +bunch of files that are specified in the default rsyslog.conf. As an example, your rsyslog.conf +might look like this: + +<pre> +# ... module loading ... +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +</pre> + +<p>Now, you want to add receive messages from a remote system and log these to +a special file, but you do not want to have these messages written to the files +specified above. The traditional approach is to add a rule in front of all others that +filters on the message, processes it and then discards it: + +<pre> +# ... module loading ... +# process remote messages +:fromhost-ip, isequal, "192.0.2.1" /var/log/remotefile +& ~ +# only messages not from 192.0.21 make it past this point + +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +</pre> + +<p>Note the tilde character, which is the discard action!. Also note that we assume that +192.0.2.1 is the sole remote sender (to keep it simple). + +<p>With multiple rulesets, we can simply define a dedicated ruleset for the remote reception +case and bind it to the receiver. This may be written as follows: + +<pre> +# ... module loading ... +# process remote messages +# define new ruleset and add rules to it: +$RuleSet remote +*.* /var/log/remotefile +# only messages not from 192.0.21 make it past this point + +# bind ruleset to tcp listener +$InputTCPServerBindRuleset remote +# and activate it: +$InputTCPServerRun 10514 + +# switch back to the default ruleset: +$RuleSet RSYSLOG_DefaultRuleset +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +</pre> + +<p>Here, we need to switch back to the default ruleset after we have defined our custom +one. This is why I recommend a different ordering, which I find more intuitive. The sample +below has it, and it leads to the same results: + +<pre> +# ... module loading ... +# at first, this is a copy of the unmodified rsyslog.conf +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +# end of the "regular" rsyslog.conf. Now come the new definitions: + +# process remote messages +# define new ruleset and add rules to it: +$RuleSet remote +*.* /var/log/remotefile + +# bind ruleset to tcp listener +$InputTCPServerBindRuleset remote +# and activate it: +$InputTCPServerRun 10514 +</pre> + +<p>Here, we do not switch back to the default ruleset, because this is not needed as it is +completely defined when we begin the "remote" ruleset. + +<p>Now look at the examples and compare them to the single-ruleset solution. You will notice +that we do <b>not</b> need a real filter in the multi-ruleset case: we can simply use +"*.*" as all messages now means all messages that are being processed by this +rule set and all of them come in via the TCP receiver! This is what makes using multiple +rulesets so much easier. + +<h3>Split local and remote logging for three different ports</h3> +<p>This example is almost like the first one, but it extends it a little bit. While it is +very similar, I hope it is different enough to provide a useful example why you may want +to have more than two rulesets. + +<p>Again, we would like to use the "regular" log files for local logging, only. But +this time we set up three syslog/tcp listeners, each one listening to a different +port (in this example 10514, 10515, and 10516). Logs received from these receivers shall go into +different files. Also, logs received from 10516 (and only from that port!) with +"mail.*" priority, shall be written into a specif file and <b>not</b> be +written to 10516's general log file. + +<p>This is the config: + +<pre> +# ... module loading ... +# at first, this is a copy of the unmodified rsyslog.conf +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +# end of the "regular" rsyslog.conf. Now come the new definitions: + +# process remote messages + +#define rulesets first +$RuleSet remote10514 +*.* /var/log/remote10514 + +$RuleSet remote10515 +*.* /var/log/remote10515 + +$RuleSet remote10516 +mail.* /var/log/mail10516 +& ~ +# note that the discard-action will prevent this messag from +# being written to the remote10516 file - as usual... +*.* /var/log/remote10516 + +# and now define listners bound to the relevant ruleset +$InputTCPServerBindRuleset remote10514 +$InputTCPServerRun 10514 + +$InputTCPServerBindRuleset remote10515 +$InputTCPServerRun 10515 + +$InputTCPServerBindRuleset remote10516 +$InputTCPServerRun 10516 +</pre> + +<p>Note that the "mail.*" rule inside the "remote10516" ruleset does +not affect processing inside any other rule set, including the default rule set. + + +<h2>Performance</h2> +<h3>Fewer Filters</h3> +<p>No rule processing can be faster than not processing a rule at all. As such, it is useful +for a high performance system to identify disjunct actions and try to split these off to +different rule sets. In the example section, we had a case where three different tcp listeners +need to write to three different files. This is a perfect example of where multiple rule sets +are easier to use and offer more performance. The performance is better simply because there +is no need to check the reception service - instead messages are automatically pushed to the +right rule set and can be processed by very simple rules (maybe even with +"*.*"-filters, the fastest ones available). + +<h3>Partitioning of Input Data</h3> +<p>Starting with rsyslog 5.3.4, rulesets permit higher concurrency. They offer the ability +to run on their own "main" queue. What that means is that a own queue is associated +with a specific rule set. That means that inputs bound to that ruleset do no longer need +to compete with each other when they enqueue a data element into the queue. Instead, enqueue +operations can be completed in parallel. +<p>An example: let us assume we have three TCP listeners. Without rulesets, each of them +needs to insert messages into the main message queue. So if each of them wants to +submit a newly arrived message into the queue at the same time, only one can do so +while the others need to wait. With multiple rulesets, its own queue can be created for each +ruleset. If now each listener is bound to its own ruleset, concurrent message submission is +possible. On a machine with a sufficiently large number of corse, this can result in +dramatic performance improvement. +<p>It is highly advised that high-performance systems define a dedicated ruleset, with a +dedicated queue for each of the inputs. +<p>By default, rulesets do <b>not</b> have their own queue. It must be activated via the +<a href="rsconf1_rulesetcreatemainqueue.html">$RulesetCreateMainQueue</a> directive. + +<h3>Future Enhancements</h3> +<p>In the long term, multiple rule sets will probably lay the foundation for even better +optimizations. So it is not a bad idea to get aquainted with them. + +<p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/omoracle.html b/doc/omoracle.html index 40f6360f..2bb6aa5d 100644 --- a/doc/omoracle.html +++ b/doc/omoracle.html @@ -13,10 +13,11 @@ <p><b>Available since: </b>: 4.3.0 <p><b>Status: </b>: contributed module, not maitained by rsyslog core authors <p><b>Description</b>:</p> -<p>This module provides native support for logging to Oracle databases. It offers -superior performance over the more generic <a href="omlibdbi.html">omlibdbi</a> module. -It also includes a number of enhancements, most importantly prepared statements and -batching, what provides a big performance improvements. +<p>This module provides native support for logging to Oracle +databases. It offers superior performance over the more +generic <a href="omlibdbi.html">omlibdbi</a> module. It also includes +a number of enhancements, most importantly prepared statements and +batching, what provides a big performance improvement. </p> <p>Note that this module is maintained by its original author. If you need assistance with it, it is suggested to post questions to the @@ -63,10 +64,132 @@ it is suggested to post questions to the $OmoracleStatement \ insert into foo(hostname,message)values(:host,:message) - Also note that identifiers to placeholders are arbitrarry. You + Also note that identifiers to placeholders are arbitrary. You need to define the properties on the template in the correct order you want them passed to the statement! </pre> +<p>Some additional documentation contributed by Ronny Egner: +<pre> +REQUIREMENTS: +-------------- + +- Oracle Instantclient 10g (NOT 11g) Base + Devel + (if you´re on 64-bit linux you should choose the 64-bit libs!) +- JDK 1.6 (not neccessary for oracle plugin but "make" didd not finsished successfully without it) + +- "oracle-instantclient-config" script + (seems to shipped with instantclient 10g Release 1 but i was unable to find it for 10g Release 2 so here it is) + + +====================== /usr/local/bin/oracle-instantclient-config ===================== +#!/bin/sh +# +# Oracle InstantClient SDK config file +# Jean-Christophe Duberga - Bordeaux 2 University +# + +# just adapt it to your environment +incdirs="-I/usr/include/oracle/10.2.0.4/client64" +libdirs="-L/usr/lib/oracle/10.2.0.4/client64/lib" + +usage="\ +Usage: oracle-instantclient-config [--prefix[=DIR]] [--exec-prefix[=DIR]] [--version] [--cflags] [--libs] [--static-libs]" + +if test $# -eq 0; then + echo "${usage}" 1>&2 + exit 1 +fi + +while test $# -gt 0; do + case "$1" in + -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; + *) optarg= ;; + esac + + case $1 in + --prefix=*) + prefix=$optarg + if test $exec_prefix_set = no ; then + exec_prefix=$optarg + fi + ;; + --prefix) + echo $prefix + ;; + --exec-prefix=*) + exec_prefix=$optarg + exec_prefix_set=yes + ;; + --exec-prefix) + echo ${exec_prefix} + ;; + --version) + echo ${version} + ;; + --cflags) + echo ${incdirs} + ;; + --libs) + echo $libdirs -lclntsh -lnnz10 -locci -lociei -locijdbc10 + ;; + --static-libs) + echo "No static libs" 1>&2 + exit 1 + ;; + *) + echo "${usage}" 1>&2 + exit 1 + ;; + esac + shift +done + +=============== END ============== + + + + +COMPILING RSYSLOGD +------------------- + + +./configure --enable-oracle + + + + +RUNNING +------- + +- make sure rsyslogd is able to locate the oracle libs (either via LD_LIBRARY_PATH or /etc/ld.so.conf) +- set TNS_ADMIN to point to your tnsnames.ora +- create a tnsnames.ora and test you are able to connect to the database + +- create user in oracle as shown in the following example: + create user syslog identified by syslog default tablespace users quota unlimited on users; + grant create session to syslog; + create role syslog_role; + grant syslog_role to syslog; + grant create table to syslog_role; + grant create sequence to syslog_role; + +- create tables as needed + +- configure rsyslog as shown in the following example + $ModLoad omoracle + + $OmoracleDBUser syslog + $OmoracleDBPassword syslog + $OmoracleDB syslog + $OmoracleBatchSize 1 + $OmoracleBatchItemSize 4096 + + $OmoracleStatementTemplate OmoracleStatement + $template OmoracleStatement,"insert into foo(hostname,message) values (:host,:message)" + $template TestStmt,"%hostname%%msg%" + *.* :omoracle:;TestStmt + (you guess it: username = password = database = "syslog".... see $rsyslogd_source/plugins/omoracle/omoracle.c for me info) +</pre> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the diff --git a/doc/omruleset.html b/doc/omruleset.html new file mode 100644 index 00000000..41d6ccfc --- /dev/null +++ b/doc/omruleset.html @@ -0,0 +1,140 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>ruleset output module (omruleset)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>ruleset output/including module (omruleset)</h1> +<p><b>Module Name: omruleset</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 5.3.4</p> +<p><b>Description</b>:</p> +<p>This is a very special "output" module. It permits to pass a message object +to another rule set. While this is a very simple action, it enables very +complex configurations, e.g. it supports high-speed "and" conditions, sending +data to the same file in a non-racy way, include-ruleset functionality as well as +some high-performance optimizations (in case the rule sets have the necessary +queue definitions). +<p>While it leads to a lot of power, this output module offers seamingly easy functionaltiy. +The complexity (and capablities) arise from how everthing can be combined. +<p>With this module, a message can be sent to processing to another ruleset. This is somewhat +similar to a "#include" in the C programming language. However, one needs to keep +on the mind that a ruleset can contain its own queue and that a queue can run in various modes. +<p>Note that if no queue is defined in the ruleset, the message is enqueued into the main message +queue. This most often is not optimal and means that message processing may be severely defered. +Also note that when the ruleset's target queue is full and no free space can be aquired +within the usual timeout, the message object may actually be lost. This is an extreme scenario, +but users building an audit-grade system need to know this restriction. For regular installations, +it should not really be relevant. +<p><b>At minimum, be sure you understand the +<a href="rsconf1_rulesetcreatemainqueue.html">$RulesetCreateMainQueue</a> +directive as well as the importance of statement order in rsyslog.conf before using omruleset!</b> +<p><b>Recommended Use:</b> +<ul> +<li>create rulesets specifically for omruleset +<li>create these rulesets with their own main queue +<li> decent queueing parameters (sizes, threads, etc) should be used +for the ruleset main queue. If in doubt, use the same parameters as for the +overall main queue. +<li>if you use multiple levels of ruleset nesting, double check for endless loops - the rsyslog +engine does not detect these +</ul> + +<p><b>Configuration Directives</b>:</p> +<ul> +<li><b>$ActionOmrulesetRulesetName</b> ruleset-to-submit-to<br> +This directive specifies the name of the ruleset that the message +provided to omruleset should be submitted to. This ruleset must already have +been defined. Note that the directive is automatically reset after each +:omruleset: action and there is no default. This is done to prevent accidential +loops in ruleset definition, what can happen very quickly. +The :omruleset: action will NOT be honored if no ruleset name has been defined. As usual, +the ruleset name must be specified in front of the action that it modifies. +</ul> +<p><b>Examples:</b></p> +<p>This example creates a ruleset for a write-to-file action. The idea here +is that the same file is written based on multiple filters, problems occur if the file is used +together with a buffer. That is because file buffers are action-specific, and so some partial +buffers would be written. With omruleset, we create a single action inside its own ruleset and +then pass all messages to it whenever we need to do so. Of course, such a simple situation could +also be solved by a more complex filter, but the method used here can also be utilized in +more complex scenarios (e.g. with multiple listeners). The example tries to keep it simple. +Note that we create a ruleset-specific main queue (for simplicity with the default main queue +parameters) in order to avoid re-queueing messages back into the main queue. +</p> +<textarea rows="30" cols="80">$ModLoad omruleset + +# define ruleset for commonly written file +$RuleSet commonAction +$RulesetCreateMainQueue on +*.* /path/to/file.log + +#switch back to default ruleset +$ruleset RSYSLOG_DefaultRuleset + +# begin first action +# note that we must first specify which ruleset to use for omruleset: +$ActionOmrulesetRulesetName CommonAction +mail.info :omruleset: +#end first action + +# begin second action +# note that we must first specify which ruleset to use for omruleset: +$ActionOmrulesetRulesetName CommonAction +:FROMHOST, isequal, "myhost.example.com" :omruleset: +#end second action + +# of course, we can have "regular" actions alongside :omrulset: actions +*.* /path/to/general-message-file.log +</textarea> +<p>The next example is used to creat a high-performance nested and filter condition. Here, +it is first checked if the message contains a string "error". If so, the message is forwarded +to another ruleset which then applies some filters. The advantage of this is that we can use +high-performance filters where we otherwise would need to use the (much slower) expression-based +filters. Also, this enables pipeline processing, in that second ruleset is executed in +parallel to the first one.</p> +<textarea rows="30" cols="80">$ModLoad omruleset + +# define "second" ruleset +$RuleSet nested +$RulesetCreateMainQueue on # again, we use our own queue +mail.* /path/to/mailerr.log +kernel.* /path/to/kernelerr.log +auth.* /path/to/autherr.log + +#switch back to default ruleset +$ruleset RSYSLOG_DefaultRuleset + +# begin first action - here we filter on "error" +# note that we must first specify which ruleset to use for omruleset: +$ActionOmrulesetRulesetName nested +:msg, contains, "error :omruleset: +#end first action + +# begin second action - as an example we can do anything else in +# this processing. Note that these actions are processed concurrently +# to the ruleset "nested" +:FROMHOST, isequal, "myhost.example.com" /path/to/host.log +#end second action + +# of course, we can have "regular" actions alongside :omrulset: actions +*.* /path/to/general-message-file.log +</textarea> +<p><b>Caveats/Known Bugs:</b> +<p>The current configuration file language is not really adequate for a complex construct +like omruleset. Unfortunately, more important work is currently preventing me from redoing the +config language. So use extreme care when nesting rulesets and be sure to test-run your +config before putting it into production, ensuring you have a suffciently large probe +of the traffic run over it. If problems arise, the +<a href="troubleshoot.html">rsyslog debug log</a> is your friend. +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/omstdout.html b/doc/omstdout.html new file mode 100644 index 00000000..0bd10cfb --- /dev/null +++ b/doc/omstdout.html @@ -0,0 +1,42 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>stdout output module (omstdout)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>stdout output module (stdout)</h1> +<p><b>Module Name: omstdout</b></p> +<p><b>Author: </b>Rainer Gerhards +<rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 4.1.6</p> +<p><b>Description</b>:</p> +<p>This module writes any messages that are passed to it to stdout. +It was developed for the rsyslog test suite. However, there may +(limited) other uses exists. Please not that we do not put too much +effort into the quality of this module as we do not expect it to +be used in real deployments. If you do, please drop us a note so +that we can enhance its priority! +<p><b>Configuration Directives</b>:</p> +<ul> +<li><b>$ActionOMStdoutArrayInterface</b> [on|<b>off</b><br> +This setting instructs omstdout to use the alternate +array based method of parameter passing. If used, the values +will be output with commas between the values but no other padding bytes. +This is a test aid for the alternate calling interface. +<li><b>$ActionOMStdoutEnsureLFEnding</b> [<b>on</b>|off<br> +Makes sure that each message is written with a terminating LF. This is needed for +the automatted tests. If the message contains a trailing LF, none is added. +</ul> +<b>Caveats/Known Bugs:</b> +<p>Currently none known. +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/omudpspoof.html b/doc/omudpspoof.html new file mode 100644 index 00000000..64f83752 --- /dev/null +++ b/doc/omudpspoof.html @@ -0,0 +1,77 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<title>UDP spoofing output module (omudpspoof)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>UDP spoofing output module (omudpspoof)</h1> +<p><b>Module Name: omstdout</b></p> +<p><b>Author: </b>David Lang <david@lang.hm> and Rainer Gerhards +<rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 5.1.3</p> +<p><b>Description</b>:</p> +<p>This module is similar to the regular UDP forwarder, but permits to +spoof the sender address. Also, it enables to circle through a number of +source ports. +<p><b>Configuration Directives</b>:</p> +<ul> +<li><b>$ActionOMUDPSpoofSourceNameTemplate</b> <templatename><br> +This MUST be specified. It is the name of the template that contains a +numerical IP address that is to be used as the source system IP address. +While it may often be a constant value, it can be generated as usual via the +property replacer, as long as it is a valid IPv4 address. +<li><b>$ActionOMUDPSpoofTargetHost</b> <hostname><br> +Host that the messages shall be sent to. +<li><b>$ActionUDPSpoofTargetPort</b> <port><br> +Remote port that the messages shall be sent to. +<li><b>$ActionOMUDPSpoofDefaultTemplate</b> <templatename><br> +This setting instructs omudpspoof to use a template different from the +default template for all of its actions that do not have a template specified +explicitely. +<li><b>$ActionOMUDPSpoofSourcePortStart</b> <number><br> +Specifies the start value for circeling the source ports. Must be less than or +equal to the end value. Default is 32000. +<li><b>$ActionOMUDPSpoofSourcePortEnd</b> <number><br> +Specifies the ending value for circeling the source ports. Must be less than or +equal to the start value. Default is 42000. +</ul> +<b>Caveats/Known Bugs:</b> +<ul> +<li><b>IPv6</b> is currently not supported. If you need this capability, please let us +know via the rsyslog mailing list. +</ul> +<p><b>Sample:</b></p> +<p>The following sample forwards all syslog messages in unmodified form to the +remote server server.example.com. The sender address 192.0.2.1 with the fixed +source port 514 is used. +</p> +<textarea rows="8" cols="80">$ModLoad omudpspoof +$template spoofaddr,"192.0.2.1" +$template spooftemplate,"%rawmsg%" +$ActionUDPSpoofSourceNameTemplate spoofaddr +$ActionUDPSpoofTargetHost server.example.com +$ActionUDPSpoofSourcePortStart 514 +$ActionUDPSpoofSourcePortEnd 514 +*.* :omudpspoof:;spooftemplate +</textarea> +<p>The following sample is similar to the first, but uses as many defaults as possible. +In that sample, a source port in the range 32000..42000 is used. The message is formatted +according to rsyslog's canned default forwarding format. Note that if any parameters +have been changed, the previously set defaults will be used! +</p> +<textarea rows="5" cols="80">$ModLoad omudpspoof +$template spoofaddr,"192.0.2.1" +$ActionUDPSpoofSourceNameTemplate spoofaddr +$ActionUDPSpoofTargetHost server.example.com +*.* :omudpspoof: +</textarea> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/professional_support.html b/doc/professional_support.html index 7724ede8..de3ac800 100644 --- a/doc/professional_support.html +++ b/doc/professional_support.html @@ -1,112 +1,27 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html><head> -<meta http-equiv="Content-Language" content="en"><title>Professional Support for Rsyslog</title> - +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>Professional Support for Rsyslog</title> </head> + <body> -<h1>Professional Services for Rsyslog</h1> +<h1><a href="http://www.rsyslog.com/Article412.phtml">Professional Services for Rsyslog</a></h1> <p>Professional services are being offered by <a href="http://www.adiscon.com">Adiscon</a>, the company -that sponsors rsyslog development. For details, please contact <a href="mailto:info%40adiscon.com">Adiscon Sales</a>. </p> +that sponsors rsyslog development. For details, please contact <a href="mailto:info%40adiscon.com">Adiscon Sales</a>.</p> -<h3>EMail Support Service</h3> -Price: 99.00 EURO <br> -Duration: 180 days -<br> -Support level: 8x5 -<p>Purchase rsyslog support directly from the source. This -contract provides priority email support. It is a great option if you -need to provide proof of software support in your organization. This -contract provides</p> -<ul> -<li>unlimited email support tickets during validity -</li> -<li><span style="font-weight: bold;">fixes for</span> -current and <span style="font-weight: bold;">past rsyslog -releases</span> -</li> -<li>advise on how to implement rsyslog in the best possible -way. -</li> -</ul> -<p>Under this contract, fixes for old rsyslog releases will be -provided / created, provided that it is possible to do that with the -code base in question. Phone support is not included.</p> -<h3>Custom-Written Config File</h3> -Price: 29.00 EURO -<br> -Duration: N/A -<br> -Support level: 8x5 -<p>Creating rsyslog config files is easy - but if you would like -to have that extra feature and have no time to do it, this service is -for you. Important: BEFORE you purchase this service, contact us and -inquire (via <a href="mailto:info%40adiscon.com">info@adiscon.com</a>) -whether or not your desired result can be achieved via rsyslog. Once -this is clear, order the service and we will ship a custom-made -configuration file within 5 working days (at latest, most often much -faster). For security reasons, we will not put passwords into the -configuration file, but will place easy to read comments in the places -where you need to put them in. The agreement is governed under German -law. You may also purchase this service if you would like to have your -own configuration file reviewed, e.g. for auditing purposes.</p> -<h3>Local Installation Support</h3> -<p>If you intend to install rsyslog on your system but would like -to do so with minimal effort and according to your specification, you -can ask us to perform the installation for you. You get a perfect -installation, exactly like you needed, but without a need to -touch the system. This is a perfect choice for the busy administrator! -<p>In order to perform this work, we just need ssh access to your -system and the proper permissions. -<p>We charge a low one-time fee for this service. For details, please -contact <a href="mailto:info@adiscon.com">info@adiscon.com</a>. -<h3>Local Installation Maintenance</h3> -<p>If you used our services to set up the system, why not keep it -running perfectly with maintenance support? Under this contract, we -assure you run a recent build that does not interfere with your -environment and we even carry out change requests you may have. So this -is a hassle-free, everything cared about solution. -<p>Again, all we need to have is ssh access and the proper permissions -to your machine. Of course, work will only be carried out when you -expect us to do so. You are always in control of what happens. This -is a perfect outsourcing solution for those who would like to run -a great logging system but can not afford the time to keep it -in perfect shape! -<p>We charge a low monthly fee for this service. For details, please -contact <a href="mailto:info@adiscon.com">info@adiscon.com</a>. -<h3>Custom Development</h3> -<p>Do you need an exotic feature that otherwise would not be implemented? -Do you need something really quick, quicker than it is available via -the regular development schedule? Then, you may consider funding -development for a specific functionality. We are always looking for -interesting projects. If you hire us to to do the job, you can be sure -to get the best possible and probably quickest solution, because we are -obviously at the heart of the source code. No need to get aquainted to -anything, no risk of misunderstanding program concepts. Benefit from -our vast syslog experience.</p> -<p>Please note that custom development is not limited to rsyslog. We offer -a number of logging solutions and can also work as part of your time -for specific requirements. The opportunities are endless, just ask. We -will work with you on your requirements and provide a quote on the -estimated cost. Just write to <a href="mailto:sales@adiscon.com">sales@adiscon.com</a> for details.</p><h3>Consulting Services</h3> -<p>Do you have demanding logging requirements? Why not talk to a -real logging professional? Instead of trying to find the solution -like a needle in the haystack, talk to the team that brought rsyslog, -phpLogCon, the Windows MonitorWare products and other logging -solutions. We sweat logging for over 15 years now and can help quickly. -Depending on your needs, consulting can be carried out via email, the -phone or on your premises (for larger or local projects). Everything is -possible, it just depends on your needs. Consulting services are -available in English and German. Just mail <a href="mailto:sales@adiscon.com">sales@adiscon.com</a> what you are interested in and we will work with you on a proposal that fits your needs. -</p><p></p><p>All agreements are -governed under German law. -</p> +<p>In an effort to keep the documentation set clean and enable Adiscon to enhance the +professional offerings, the actual +<a href="http://www.rsyslog.com/Article412.phtml">rsyslog service offering</a> +has been moved to a +<a href="http://www.rsyslog.com/Article412.phtml">separate page</a>. +Please follow that link for details. <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> -<p><font size="2">This documentation is part of the -<a href="http://www.rsyslog.com/">rsyslog</a> +<p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html> +</body> +</html> diff --git a/doc/property_replacer.html b/doc/property_replacer.html index a6e9b518..7b604ea0 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -30,10 +30,6 @@ Currently supported are:</p> socket. Should be useful for debugging.</td> </tr> <tr> -<td><b>uxtradmsg</b></td> -<td>will disappear soon - do NOT use!</td> -</tr> -<tr> <td><b>hostname</b></td> <td>hostname from the message</td> </tr> diff --git a/doc/queue_msg_state.dot b/doc/queue_msg_state.dot new file mode 100644 index 00000000..bfef2657 --- /dev/null +++ b/doc/queue_msg_state.dot @@ -0,0 +1,25 @@ +// This file is part of rsyslog. +// +// rsyslog message state in queue processing +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + rankdir=LR + + prod [label="producer" style="dotted" shape="box"] + que [label="queued"] + deq [label="dequeued"] + del [label="deleted"] + + prod -> que [label="qEnq()" style="dotted"] + que -> deq [label="qDeq()"] + deq -> del [label="qDel()"] + deq -> que [label="fatal failure\n& restart"] + + //{rank=same; del apf pdn } +} diff --git a/doc/queue_msg_state.jpeg b/doc/queue_msg_state.jpeg Binary files differnew file mode 100644 index 00000000..a215f000 --- /dev/null +++ b/doc/queue_msg_state.jpeg diff --git a/doc/queues.html b/doc/queues.html index 4a9509a0..75b70fbf 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -115,7 +115,11 @@ isolation. This is currently selected by specifying different <i>$WorkDirectory< config directives before the queue creation statement.</p> <p>To create a disk queue, use the "<i>$<object>QueueType Disk</i>" config directive. Checkpoint intervals can be specified via "<i>$<object>QueueCheckpointInterval</i>", -with 0 meaning no checkpoints. </p> +with 0 meaning no checkpoints. Note that disk-based queues can be made very reliable +by issuing a (f)sync after each write operation. Starting with version 4.3.2, this can +be requested via "<i><object>QueueSyncQueueFiles on/off</i> with the +default being off. Activating this option has a performance penalty, so it should +not be turned on without reason.</p> <h2>In-Memory Queues</h2> <p>In-memory queue mode is what most people have on their mind when they think about computing queues. Here, the enqueued data elements are held in memory. @@ -332,6 +336,33 @@ in this regard - it was just not requested so far. So if you need more fine-grained control, let us know and we'll probably implement it. There are two configuration directives, both should be used together or results are unpredictable:" <i>$<object>QueueDequeueTimeBegin <hour></i>" and "<i>$<object>QueueDequeueTimeEnd <hour></i>". The hour parameter must be specified in 24-hour format (so 10pm is 22). A use case for this parameter can be found in the <a href="http://wiki.rsyslog.com/index.php/OffPeakHours">rsyslog wiki</a>. </p> +<h2>Performance</h2> +<p>The locking involved with maintaining the queue has a potentially large +performance impact. How large this is, and if it exists at all, depends much on +the configuration and actual use case. However, the queue is able to work on +so-called "batches" when dequeueing data elements. With batches, +multiple data elements are dequeued at once (with a single locking call). +The queue dequeues all available elements up to a configured upper +limit (<i><object>DequeueBatchSize <number></i>). It is important +to note that the actual upper limit is dictated by availability. The queue engine +will never wait for a batch to fill. So even if a high upper limit is configured, +batches may consist of fewer elements, even just one, if there are no more elements +waiting in the queue. +<p>Batching +can improve performance considerably. Note, however, that it affects the +order in which messages are passed to the queue worker threads, as each worker +now receive as batch of messages. Also, the larger the batch size and the higher +the maximum number of permitted worker threads, the more main memory is needed. +For a busy server, large batch sizes (around 1,000 or even more elements) may be useful. +Please note that with batching, the main memory must hold BatchSize * NumOfWorkers +objects in memory (worst-case scenario), even if running in disk-only mode. So if you +use the default 5 workers at the main message queue and set the batch size to 1,000, you need +to be prepared that the main message queue holds up to 5,000 messages in main memory +<b>in addition</b> to the configured queue size limits! +<p>The queue object's default maximum batch size +is eight, but there exists different defaults for the actual parts of +rsyslog processing that utilize queues. So you need to check these object's +defaults. <h2>Terminating Queues</h2> <p>Terminating a process sounds easy, but can be complex. Terminating a running queue is in fact the most complex operation a queue diff --git a/doc/rainerscript.html b/doc/rainerscript.html index ef0e41cb..63a79040 100644 --- a/doc/rainerscript.html +++ b/doc/rainerscript.html @@ -51,13 +51,25 @@ of a and b should be tested as "a <> b". The "not" operator should be reserved to cases where it actually is needed to form a complex boolean expression. In those cases, parenthesis are highly recommended. +<h2>Functions</h2> +<p>RainerScript supports a currently quite limited set of functions: +<ul> +<li>getenv(str) - like the OS call, returns the value of the environment +variable, if it exists. Returns an empty string if it does not exist. +<li>strlen(str) - returns the length of the provided string +<li>tolower(str) - converts the provided string into lowercase +</ul> +<p>The following example can be used to build a dynamic filter based on some environment +variable: +<pre> +if $msg contains getenv('TRIGGERVAR') then /path/to/errfile +</pre> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html>
\ No newline at end of file +</body></html> diff --git a/doc/rfc5424layers.png b/doc/rfc5424layers.png Binary files differnew file mode 100644 index 00000000..70192cc0 --- /dev/null +++ b/doc/rfc5424layers.png diff --git a/doc/rsconf1_abortonuncleanconfig.html b/doc/rsconf1_abortonuncleanconfig.html new file mode 100644 index 00000000..77526c07 --- /dev/null +++ b/doc/rsconf1_abortonuncleanconfig.html @@ -0,0 +1,37 @@ +<html> +<head> +<title>rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">rsyslog.conf configuration directive</a> + +<h2>$AboortOnUncleanConfig</h2> +<p><b>Type:</b> global configuration directive</p> +<p><b>Parameter Values:</b> boolean (on/off, yes/no)</p> +<p><b>Available since:</b> 5.3.1+</p> +<p><b>Default:</b> off</p> +<p><b>Description:</b></p> +<p>This directive permits to prevent rsyslog from running when the configuration file +is not clean. "Not Clean" means there are errors or some other annoyances that rsyslgod +reports on startup. This is a user-requested feature to have a strict startup mode. Note +that with the current code base it is not always possible to differentiate between an +real error and a warning-like condition. As such, the startup will also prevented if +warnings are present. I consider this a good thing in being "strict", but I admit +there also currently is no other way of doing it. +<p><b>Caveats:</b></p> +Note that the consequences of a failed rsyslogd startup can be much more serious than a +startup with only partial configuration. For example, log data may be lost or systems that +depend on the log server in question will not be able to send logs, what in the ultimate +result could result in a system hang on those systems. Also, the local system may hang when +the local log socket has become full and is not read. There exist many such scenarios. +As such, it is strongly recommended not to turn on this directive. + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_dircreatemode.html b/doc/rsconf1_dircreatemode.html index 9a9c61eb..b22b6c59 100644 --- a/doc/rsconf1_dircreatemode.html +++ b/doc/rsconf1_dircreatemode.html @@ -7,9 +7,13 @@ <h2>$DirCreateMode</h2> <p><b>Type:</b> global configuration directive</p> -<p><b>Default:</b> 0644</p> +<p><b>Default:</b> 0700</p> <p><b>Description:</b></p> <p>This is the same as $FileCreateMode, but for directories automatically generated.</p> +<p>Please visit the +<a target="_blank" href="http://lists.adiscon.net/pipermail/rsyslog/2009-April/001986.html">rsyslog mailing list +archive</a> +to understand why the default is so restrictive.</p> <p><b>Sample:</b></p> <p><code><b></b></code></p> @@ -17,8 +21,8 @@ index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2007-2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/rsconf1_dirgroup.html b/doc/rsconf1_dirgroup.html index de070126..4bc8692f 100644 --- a/doc/rsconf1_dirgroup.html +++ b/doc/rsconf1_dirgroup.html @@ -9,7 +9,7 @@ <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> <p><b>Description:</b></p> -<p>Set the group for directories newly created. Please note that this setting does not affect the group of directories already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd on startup and on HUPing. Interim changes to the user mapping are not detected.</p> +<p>Set the group for directories newly created. Please note that this setting does not affect the group of directories already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd on during startup processing. Interim changes to the user mapping are not detected.</p> <p><b>Sample:</b></p> <p><code><b>$DirGroup loggroup</b></code></p> diff --git a/doc/rsconf1_dirowner.html b/doc/rsconf1_dirowner.html index da8e252d..f779c008 100644 --- a/doc/rsconf1_dirowner.html +++ b/doc/rsconf1_dirowner.html @@ -9,7 +9,7 @@ <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> <p><b>Description:</b></p> -<p>Set the file owner for directories newly created. Please note that this setting does not affect the owner of directories already existing. The parameter is a user name, for which the userid is obtained by rsyslogd on startup and on HUPing. Interim changes to the user mapping are not detected.</p> +<p>Set the file owner for directories newly created. Please note that this setting does not affect the owner of directories already existing. The parameter is a user name, for which the userid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.</p> <p><b>Sample:</b></p> <p><code><b>$DirOwner loguser</b></code></p> diff --git a/doc/rsconf1_escape8bitcharsonreceive.html b/doc/rsconf1_escape8bitcharsonreceive.html new file mode 100644 index 00000000..408851c1 --- /dev/null +++ b/doc/rsconf1_escape8bitcharsonreceive.html @@ -0,0 +1,44 @@ +<html> +<head> +<title>rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">back</a> + +<h2>$Escape8BitCharactersOnReceive</h2> +<p><b>Type:</b> global configuration directive</p> +<p><b>Default:</b> off</p> +<p><b>Available Since:</b> 5.5.2</p> +<p><b>Description:</b></p> +<p>This directive instructs rsyslogd to replace non US-ASCII characters (those that +have the 8th bit set) during reception of the message. +This may be useful for some systems. +Please note that this escaping breaks Unicode and many other encodings. Most importantly, +it can be assumed that Asian and European characters will be rendered hardly readable by +this settings. However, it may still be useful when the logs themself are primarily +in English and only occasionally contain local script. +If this option is turned on, all control-characters are converted to a 3-digit octal number and be prefixed with the $ControlCharacterEscapePrefix character (being '#' by default). +<p><b>Warning:</b></p> +<ul> + <li>turning on this option most probably destroys non-western character sets + (like Japanese, Chinese and Korean) as well as European character sets.</li> + <li>turning on this option destroys digital signatures if such exists inside + the message</li> + <li>if turned on, the drop-cc, space-cc and escape-cc + <a href="property_replacer.html">property replacer</a> options do not work + as expected because control characters are already removed upon message + reception. If you intend to use these property replacer options, you must + turn off $Escape8BitCharactersOnReceive.</li> +</ul> +<p><b>Sample:</b></p> +<p><code><b>$Escape8BitCharactersOnReceive on</b></code></p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_filegroup.html b/doc/rsconf1_filegroup.html index dd5b8ad5..935f074a 100644 --- a/doc/rsconf1_filegroup.html +++ b/doc/rsconf1_filegroup.html @@ -9,7 +9,7 @@ <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> <p><b>Description:</b></p> -<p>Set the group for dynaFiles newly created. Please note that this setting does not affect the group of files already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd on startup and on HUPing. Interim changes to the user mapping are not detected.</p> +<p>Set the group for dynaFiles newly created. Please note that this setting does not affect the group of files already existing. The parameter is a group name, for which the groupid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.</p> <p><b>Sample:</b></p> <p><code><b>$FileGroup loggroup</b></code></p> diff --git a/doc/rsconf1_fileowner.html b/doc/rsconf1_fileowner.html index 935cfffd..62125c8d 100644 --- a/doc/rsconf1_fileowner.html +++ b/doc/rsconf1_fileowner.html @@ -9,7 +9,7 @@ <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> <p><b>Description:</b></p> -<p>Set the file owner for dynaFiles newly created. Please note that this setting does not affect the owner of files already existing. The parameter is a user name, for which the userid is obtained by rsyslogd on startup and on HUPing. Interim changes to the user mapping are not detected.</p> +<p>Set the file owner for dynaFiles newly created. Please note that this setting does not affect the owner of files already existing. The parameter is a user name, for which the userid is obtained by rsyslogd during startup processing. Interim changes to the user mapping are not detected.</p> <p><b>Sample:</b></p> <p><code><b>$FileOwner loguser</b></code></p> diff --git a/doc/rsconf1_markmessageperiod.html b/doc/rsconf1_markmessageperiod.html index 2c833339..a6486ba1 100644 --- a/doc/rsconf1_markmessageperiod.html +++ b/doc/rsconf1_markmessageperiod.html @@ -7,7 +7,7 @@ <h2>$MarkMessagePeriod</h2> <p><b>Type:</b> specific to immark input module</p> -<p><b>Default:</b> 1800 (20 minutes)</p> +<p><b>Default:</b> 1200 (20 minutes)</p> <p><b>Description:</b></p> <p>This specifies when mark messages are to be written to output modules. The time specified is in seconds. Specifying 0 is possible and disables mark diff --git a/doc/rsconf1_omfileforcechown.html b/doc/rsconf1_omfileforcechown.html new file mode 100644 index 00000000..7415a6f6 --- /dev/null +++ b/doc/rsconf1_omfileforcechown.html @@ -0,0 +1,64 @@ +<html> +<head> +<title>rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">back</a> + +<h2>$omfileForceChown</h2> +<p><b>Type:</b> global configuration directive</p> +<p><b>Parameter Values:</b> boolean (on/off, yes/no)</p> +<p><b>Available since:</b> 4.7.0+, 5.3.0+</p> +<p><b>Default:</b> off</p> +<p><b>Description:</b></p> +<p>Forces rsyslogd to change the ownership for output files that already exist. Please note +that this tries to fix a potential problem that exists outside the scope of rsyslog. Actually, +it tries to fix invalid ownership/permission settings set by the original file creator. +<p>Rsyslog changes the ownership during initial execution with root privileges. When a privelege +drop is configured, privileges are dropped after the file owner ship is changed. Not that this currently +is a limitation in rsyslog's privilege drop code, which is on the TODO list to be removed. See Caveats +section below for the important implications. +<p><b>Caveats:</b></p> +<p>This directive tries to fix a problem that actually is outside the scope of rsyslog. As such, +there are a couple of restrictions and situations in which it will not work. <b>Users are strongly +encouraged to fix their system instead of turning this directive on</b> - it should only be used +as a last resort. +<p>At least in the following scenario, this directive will fail expectedly: +<p>It does not address +the situation that someone changes the ownership *after* rsyslogd has started. +Let's, for example, consider a log rotation script. +<ul> +<li>rsyslog is started +<li>ownership is changed +<li>privileges dropped +<li>log rotation (lr) script starts +<li>lr removes files +<li>lr creates new files with root:adm (or whatever else) +<li>lr HUPs rsyslogd +<li>rsyslogd closes files +<li>rsyslogd tries to open files +<li>rsyslogd tries to change ownership --> fail as we are non-root now +<li>file open fails +</ul> + +Please note that once the privilege drop code is refactored, this directive will +no longer work, because then privileges will be dropped before any action is performed, +and thus we will no longer be able to chown files that do not belong to the +user rsyslogd is configured to run under. + +<p>So <b>expect the directive to go away</b>. It will not +be removed in version 4, but may disappear at any time for any version greater than 4. + +<p><b>Sample:</b></p> +<p><code><b>$FileOwner loguser</b> +<br><b>$omfileForceChown on</b></code></p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_rulesetcreatemainqueue.html b/doc/rsconf1_rulesetcreatemainqueue.html new file mode 100644 index 00000000..5c1e0dec --- /dev/null +++ b/doc/rsconf1_rulesetcreatemainqueue.html @@ -0,0 +1,83 @@ +<html> +<head> +<title>RulesetCreateMainQueue - rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">rsyslog.conf configuration directive</a> + +<h2>$RulesetCreateMainQueue</h2> +<p><b>Type:</b> ruleset-specific configuration directive</p> +<p><b>Parameter Values:</b> boolean (on/off, yes/no)</p> +<p><b>Available since:</b> 5.3.5+</p> +<p><b>Default:</b> off</p> +<p><b>Description:</b></p> +<p> +Rulesets may use their own "main" message queue for message submission. Specifying +this directive, <b>inside a ruleset definition</b>, turns this on. This is both a performance +enhancement and also permits different rulesets (and thus different inputs within the same +rsyslogd instance) to use different types of main message queues. +<p>The ruleset queue is created with the parameters that are specified for the main message +queue at the time the directive is given. If different queue configurations are desired, +different main message queue directives must be used in front of the $RulesetCreateMainQueue +directive. Note that this directive may only be given once per ruleset. If multiple statements +are specified, only the first is used and for the others error messages are emitted. +<p>Note that the final set of ruleset configuration directives specifies the parameters for +the default main message queue. +<p>To learn more about this feature, please be sure to read about +<a href="multi_ruleset.html">multi-ruleset support in rsyslog</a>. +<p><b>Caveats:</b></p> +The configuration statement "$RulesetCreateMainQueue off" has no effect at all. +The capability to specify this is an artifact of the current (ugly!) configuration +language. + +<p><b>Example:</b></p> +<p>This example sets up a tcp server with three listeners. Each of these +three listener is bound to a specific ruleset. As a performance optimization, +the rulesets all receive their own private queue. The result is that received messages +can be independently processed. With only a single main message queue, we would have +some lock contention between the messages. This does not happen here. Note that in this +example, we use different processing. Of course, all messages could also have been +processed in the same way ($IncludeConfig may be useful in that case!). +</p> +<textarea rows="30" cols="60">$ModLoad imtcp +# at first, this is a copy of the unmodified rsyslog.conf +#define rulesets first +$RuleSet remote10514 +$RulesetCreateMainQueue on # create ruleset-specific queue +*.* /var/log/remote10514 + +$RuleSet remote10515 +$RulesetCreateMainQueue on # create ruleset-specific queue +*.* /var/log/remote10515 + +$RuleSet remote10516 +$RulesetCreateMainQueue on # create ruleset-specific queue +mail.* /var/log/mail10516 +& ~ +# note that the discard-action will prevent this messag from +# being written to the remote10516 file - as usual... +*.* /var/log/remote10516 + +# and now define listners bound to the relevant ruleset +$InputTCPServerBindRuleset remote10514 +$InputTCPServerRun 10514 + +$InputTCPServerBindRuleset remote10515 +$InputTCPServerRun 10515 + +$InputTCPServerBindRuleset remote10516 +$InputTCPServerRun 10516 +</textarea> +<p>Note the positions of the directives. With the current config language, +position is very important. This is ugly, but unfortunately the way it currently +works. +</p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_rulesetparser.html b/doc/rsconf1_rulesetparser.html new file mode 100644 index 00000000..84350431 --- /dev/null +++ b/doc/rsconf1_rulesetparser.html @@ -0,0 +1,119 @@ +<html> +<head> +<title>RulesetParser - rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">rsyslog.conf configuration directive</a> + +<h2>$RulesetParser</h2> +<p><b>Type:</b> ruleset-specific configuration directive</p> +<p><b>Parameter Values:</b> string</p> +<p><b>Available since:</b> 5.3.4+</p> +<p><b>Default:</b> rsyslog.rfc5424;rsyslog.rfc5425</p> +<p><b>Description:</b></p> +<p> +This directive permits to specify which +<a href="messageparser.html">message parsers</a> should be used for the ruleset +in question. It no ruleset is explicitely specified, the default ruleset is used. Message +parsers are contained in (loadable) parser modules with the most common cases +(RFC3164 and RFC5424) being build-in into rsyslogd. +<p>When this directive is specified the first time for a ruleset, it will not only add the +parser to the ruleset's parser chain, it will also wipe out the default parser chain. +So if you need to have +them in addition to the custom parser, you need to specify those as well. +<p>Order of directives is important. Parsers are tried one after another, in the order +they are specified inside the config. As soon as a parser is able to parse the message, +it will do so and no other parsers will be executed. If no matching parser can be found, +the message will be discarded and a warning message be issued (but only for the first +1,000 instances of this problem, to prevent message generation loops). +<p>Note that the rfc3164 parser will <b>always</b> be able to parse a message - it may +just not be the format that you like. This has two important implications: 1) always place +that parser at the END of the parser list, or the other parsers after it will never +be tried and 2) if you would like to make sure no message is lost, placing the rfc3164 +parser at the end of the parser list ensures that. +<p>Multiple parser modules are very useful if you have various devices that emit +messages that are malformed in various ways. The route to take then is +<ul> +<li>make sure you find a custom parser for that device; if there is no one, you +may consider writing one yourself (it is not that hard) or getting one written +as part of +<a href="http://www.rsyslog.com/professional-servcies">Adiscon's professional services +for rsyslog</a>. +<li>load your custom parsers via $ModLoad +<li>create a ruleset for each malformed format; assign the custom parser to it +<li>create a specific listening port for all devices that emit the same +malformed format +<li>bind the listener to the ruleset with the required parser +</ul> +<p>Note that it may be cumbersome to add all rules to all rulesets. To avoid this, +you can either use $Include or <a href="omruleset.html">omruleset</a> +(what probably provides the best solution). +<p>More information about rulesets in general can be found in +<a href="multi_ruleset.html">multi-ruleset support in rsyslog</a>. +<p><b>Caveats:</b></p> +<p>currently none known</p> + +<p><b>Example:</b></p> +<p>This example assumes there are two devices emiting malformed messages via UDP. +We have two custom parsers for them, named "device1.parser" and +"device2.parser". In addition to that, we have a number of other +devices sending wellformed messages, also via UDP. +<p>The solution is to listen for data from the two devices on two special +ports (10514 and 10515 in this example), create a ruleset for each and +assign the custom parsers to them. The rest of the messages are received via +port 514 using the regular parsers. Processing shall be equal for all messages. +So we simply forward the malformed messages to the regular queue once they are parsed (keep +in mind that a message is never again parsed once any parser properly processed it). +</p> +<textarea rows="40" cols="80">$ModLoad imudp +$ModLoad pmdevice1 # load parser "device1.parser" for device 1 +$ModLoad pmdevice2 # load parser "device2.parser" for device 2 + +# define ruleset for the first device sending malformed data +$Ruleset maldev1 +$RulesetCreateMainQueue on # create ruleset-specific queue +$RulesetParser "device1.parser" # note: this deactivates the default parsers +# forward all messages to default ruleset: +$ActionOmrulesetRulesetName RSYSLOG_DefaultRuleset +*.* :omruleset: + +# define ruleset for the second device sending malformed data +$Ruleset maldev2 +$RulesetCreateMainQueue on # create ruleset-specific queue +$RulesetParser "device2.parser" # note: this deactivates the default parsers +# forward all messages to default ruleset: +$ActionOmrulesetRulesetName RSYSLOG_DefaultRuleset +*.* :omruleset: + +# switch back to default ruleset +$Ruleset RSYSLOG_DefaultRuleset +*.* /path/to/file +auth.info @authlogger.example.net +# whatever else you usually do... + + +# now define the inputs and bind them to the rulesets +# first the default listener (utilizing the default ruleset) +$UDPServerRun 514 + +# now the one with the parser for device type 1: +$InputUDPServerBindRuleset maldev1 +$UDPServerRun 10514 + +# and finally the one for device type 2: +$InputUDPServerBindRuleset maldev2 +$UDPServerRun 10515 +</textarea> +<p>Note the positions of the directives. With the current config language, +<b>sequence of statements is very important</b>. This is ugly, but unfortunately +the way it currently works. +</p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsyslog-vers.png b/doc/rsyslog-vers.png Binary files differnew file mode 100644 index 00000000..e8ec8b81 --- /dev/null +++ b/doc/rsyslog-vers.png diff --git a/doc/rsyslog_conf_actions.html b/doc/rsyslog_conf_actions.html index 2ef3f4b0..8c4b9cfc 100644 --- a/doc/rsyslog_conf_actions.html +++ b/doc/rsyslog_conf_actions.html @@ -98,18 +98,14 @@ done, same with /dev/console.</p> messages to a remote host running rsyslogd(8) and to receive messages from remote hosts. Using this feature you're able to control all syslog messages on one host, if all other machines will log remotely to that. -This tears down<br> -administration needs.<br> -<br> -<b>Please note that this version of rsyslogd by default does NOT -forward messages it has received from the network to another host. -Specify the "-h" option to enable this.</b></p> +This tears down administration needs.</p> <p>To forward messages to another host, prepend the hostname with the at sign ("@"). A single at sign means that messages will be forwarded via UDP protocol (the standard for syslog). If you prepend two at signs ("@@"), the messages will be transmitted via TCP. Please note that plain TCP based syslog is not officially standardized, but -most major syslogds support it (e.g. syslog-ng or WinSyslog). The +most major syslogds support it (e.g. syslog-ng or +<a href="http://www.winsyslog.com/">WinSyslog</a>). The forwarding action indicator (at-sign) can be followed by one or more options. If they are given, they must be immediately (without a space) following the final at sign and be enclosed in parenthesis. The diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html index 1d30d8ae..841ec9c7 100644 --- a/doc/rsyslog_conf_filter.html +++ b/doc/rsyslog_conf_filter.html @@ -48,8 +48,8 @@ in rsyslog and offer the best performance for this job.</p> facility and a priority, separated by a period (".''). Both parts are case insensitive and can also be specified as decimal numbers, but don't do that, you have been warned. Both facilities and priorities are -described in rsyslog(3). The names mentioned below correspond to the -similar LOG_-values in /usr/include/rsyslog.h.<br> +described in syslog(3). The names mentioned below correspond to the +similar LOG_-values in /usr/include/syslog.h.<br> <br> The facility is one of the following keywords: auth, authpriv, cron, daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 43eacc43..3e926db1 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -17,6 +17,8 @@ appear as implementation progresses. many parameter settings modify queue parameters. If in doubt, use the default, it is usually well-chosen and applicable in most cases.</p> <ul> +<li><a href="rsconf1_abortonuncleanconfig.html">$AbortOnUncleanConfig</a> - abort startup if there is +any issue with the config file</li> <li><a href="rsconf1_actionexeconlywhenpreviousissuspended.html">$ActionExecOnlyWhenPreviousIsSuspended</a></li> <li>$ActionName <a_single_word> - used primarily for documentation, e.g. when generating a configuration graph. Available sice 4.3.1. @@ -58,6 +60,7 @@ default template for UDP and plain TCP forwarding action</li> <li>$ActionGSSForwardDefaultTemplate [templateName] - sets a new default template for GSS-API forwarding action</li> <li>$ActionQueueCheckpointInterval <number></li> +<li>$ActionQueueDequeueBatchSize <number> [default 16]</li> <li>$ActionQueueDequeueSlowdown <number> [number is timeout in <i> micro</i>seconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]</li> @@ -91,11 +94,32 @@ default 60000 (1 minute)]</li> <li><a href="rsconf1_actionresumeinterval.html">$ActionResumeInterval</a></li> <li>$ActionResumeRetryCount <number> [default 0, -1 means eternal]</li> <li>$ActionSendResendLastMsgOnReconn <[on/<b>off</b>]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. -<li>$ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action -</li><li>$ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver -(driver-specific)</li><li>$ActionSendStreamDriverAuthMode <mode>, authentication mode to use with the stream driver -(driver-specific)</li><li>$ActionSendStreamDriverPermittedPeer <ID>, accepted fingerprint (SHA1) or name of remote peer +<li>$ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action</li> +<li>$ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver (driver-specific)</li> +<li>$ActionSendStreamDriverAuthMode <mode>, authentication mode to use with the stream driver. Note that this directive requires TLS +netstream drivers. For all others, it will be ignored. +(driver-specific)</li> +<li>$ActionSendStreamDriverPermittedPeer <ID>, accepted fingerprint (SHA1) or name of remote peer. Note that this directive requires TLS +netstream drivers. For all others, it will be ignored. (driver-specific) -<span style="font-weight: bold;"> directive may go away</span>!</li> +<li><b>$ActionSendTCPRebindInterval</b> nbr</a>- [available since 4.5.1] - instructs the TCP send +action to close and re-open the connection to the remote host every nbr of messages sent. +Zero, the default, means that no such processing is done. This directive is useful for +use with load-balancers. Note that there is some performance overhead associated with it, +so it is advisable to not too often "rebind" the connection (what +"too often" actually means depends on your configuration, a rule of thumb is +that it should be not be much more often than once per second).</li> +<li><b>$ActionSendUDPRebindInterval</b> nbr</a>- [available since 4.3.2] - instructs the UDP send +action to rebind the send socket every nbr of messages sent. Zero, the default, means +that no rebind is done. This directive is useful for use with load-balancers.</li> +<li><b>$ActionWriteAllMarkMessages</b> [on/<b>off</b>]- [available since 5.1.5] - normally, mark messages +are written to actions only if the action was not recently executed (by default, recently means within the +past 20 minutes). If this setting is switched to "on", mark messages are always sent to actions, +no matter how recently they have been executed. In this mode, mark messages can be used as a kind of +heartbeat. Note that this option auto-resets to "off", so if you intend to use it with multiple +actions, it must be specified in front off <b>all</b> selector lines that should provide this +functionality. +</li> <li><a href="rsconf1_allowedsender.html">$AllowedSender</a></li> <li><a href="rsconf1_controlcharacterescapeprefix.html">$ControlCharacterEscapePrefix</a></li> <li><a href="rsconf1_debugprintcfsyslinehandlerlist.html">$DebugPrintCFSyslineHandlerList</a></li> @@ -105,6 +129,10 @@ default 60000 (1 minute)]</li> <li>$DefaultNetstreamDriver <drivername>, the default <a href="netstream.html">network stream driver</a> to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem></li> <li>$DefaultNetstreamDriverCertFile </path/to/certfile.pem></li> <li>$DefaultNetstreamDriverKeyFile </path/to/keyfile.pem></li> +<li><b>$DefaultRuleset</b> <i>name</i> - changes the default ruleset for unbound inputs to +the provided <i>name</i> (the default default ruleset is named +"RSYSLOG_DefaultRuleset"). It is advised to also read +our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a>.</li> <li><b>$CreateDirs</b> [<b>on</b>/off] - create directories on an as-needed basis</li> <li><a href="rsconf1_dircreatemode.html">$DirCreateMode</a></li> <li><a href="rsconf1_dirgroup.html">$DirGroup</a></li> @@ -112,7 +140,9 @@ default 60000 (1 minute)]</li> <li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li> <li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li> <li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li> +<li><a href="rsconf1_escape8bitcharsonreceive.html">$Escape8BitCharactersOnReceive</a></li> <li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> +<li><b>$EscapeControlCharactersOnReceive</b> [<b>on</b>|off] - escape USASCII HT character</li> <li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> <li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li> <li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li> @@ -122,12 +152,22 @@ default 60000 (1 minute)]</li> <li><a href="rsconf1_gssforwardservicename.html">$GssForwardServiceName</a></li> <li><a href="rsconf1_gsslistenservicename.html">$GssListenServiceName</a></li> <li><a href="rsconf1_gssmode.html">$GssMode</a></li> -<li>$HUPisRestart [<b>on</b>/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending +<li>$HUPisRestart [on/<b>off</b>] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is -not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for -compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually -the only one that is needed e.g. for log rotation. <b>It is recommended to set the setting to "off".</b></li> +not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default (since 4.5.1) is "off". +If it is set to "off", a HUP will only close open files. This is a much quicker action and usually +the only one that is needed e.g. for log rotation. <b>Restart-type HUPs (value "on") are depricated</b> +and will go away in rsyslog v5. So it is a good idea to change anything that needs it, now. +Usually that should not be a big issue, as the restart-type HUP can easily be replaced by +something along the lines of "/etc/init.d/rsyslog restart". +</li> <li><a href="rsconf1_includeconfig.html">$IncludeConfig</a></li><li>MainMsgQueueCheckpointInterval <number></li> +<li><b>$LogRSyslogStatusMessages</b> [<b>on</b>/off] - If set to on (the default), +rsyslog emits message on startup and shutdown as well as when it is HUPed. +This information might be needed by some log analyzers. If set to off, no such +status messages are logged, what may be useful for other scenarios. +[available since 4.7.0 and 5.3.0] +<li><b>$MainMsgQueueDequeueBatchSize</b> <number> [default 32]</li> <li>$MainMsgQueueDequeueSlowdown <number> [number is timeout in <i> micro</i>seconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]</li> @@ -186,6 +226,21 @@ supported in order to be compliant to the upcoming new syslog RFC series. <li><a href="rsconf1_maxopenfiles.html">$MaxOpenFiles</a></li> <li><a href="rsconf1_moddir.html">$ModDir</a></li> <li><a href="rsconf1_modload.html">$ModLoad</a></li> +<li><b>$OMFileZipLevel</b> 0..9 [default 0] - if greater 0, turns on gzip compression +of the output file. The higher the number, the better the compression, but also the +more CPU is required for zipping.</li> +<li><b>$OMFileIOBufferSize</b> <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)</li> +<li><b>$OMFileFlushOnTXEnd</b> <[<b>on</b>/off]>, default on. Omfile has the +capability to +writes output using a buffered writer. Disk writes are only done when the buffer is +full. So if an error happens during that write, data is potentially lost. In cases where +this is unacceptable, set $OMFileFlushOnTXEnd to on. Then, data is written at the end +of each transaction (for pre-v5 this means after <b>each</b> log message) and the usual +error recovery thus can handle write errors without data loss. Note that this option +severely reduces the effect of zip compression and should be switched to off +for that use case. Note that the default -off- is primarily an aid to preserve +the traditional syslogd behaviour.</li> +<li><a href="rsconf1_omfileforcechown.html">$omfileForceChown</a> - force ownership change for all files</li> <li><b>$RepeatedMsgContainsOriginalMsg</b> [on/<b>off</b>] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most @@ -194,26 +249,23 @@ line is that n is large enough to get a good idea which message was repeated but large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.</li> <li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li> <li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li> +<li><b>$Ruleset</b> <i>name</i> - starts a new ruleset or switches back to one already defined. +All following actions belong to that new rule set. +the <i>name</i> does not yet exist, it is created. To switch back to rsyslog's +default ruleset, specify "RSYSLOG_DefaultRuleset") as the name. +All following actions belong to that new rule set. It is advised to also read +our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a>.</li> +<li><b><a href="rsconf1_rulesetcreatemainqueue.html">$RulesetCreateMainQueue</a></b> on - creates +a ruleset-specific main queue. +<li><b><a href="rsconf1_rulesetparser.html">$RulesetParser</a></b> - enables to set +a specific (list of) message parsers to be used with the ruleset. <li><b>$OptimizeForUniprocessor</b> [on/<b>off</b>] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The -default may change as uniprocessor systems become less common.</li> +default may change as uniprocessor systems become less common. [available since 4.1.0]</li> <li>$PreserveFQDN [on/<b>off</b>) - if set to off (legacy default to remain compatible to sysklogd), the domain part from a name that is within the same domain as the receiving system is stripped. If set to on, full names are always used.</li> <li>$WorkDirectory <name> (directory for spool and other work files)</li> -<li>$UDPServerAddress <IP> (imudp) -- local IP -address (or name) the UDP listens should bind to</li> -<li>$UDPServerRun <port> (imudp) -- former --r<port> option, default 514, start UDP server on this -port, "*" means all addresses</li> -<li>$UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance -optimization. Getting the system time is very costly. With this setting, imudp can -be instructed to obtain the precise time only once every n-times. This logic is -only activated if messages come in at a very fast rate, so doing less frequent -time calls should usually be acceptable. The default value is two, because we have -seen that even without optimization the kernel often returns twice the identical time. -You can set this value as high as you like, but do so at your own risk. The higher -the value, the less precise the timestamp. <li><a href="droppriv.html">$PrivDropToGroup</a></li> <li><a href="droppriv.html">$PrivDropToGroupID</a></li> <li><a href="droppriv.html">$PrivDropToUser</a></li> diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index df9abeea..a246d0ca 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -2,15 +2,55 @@ <html><head><title>Modules - rsyslog.conf</title></head> <body> <p>This is a part of the rsyslog.conf documentation.</p> -<a href="rsyslog_conf.html">back</a> -<h2>Modules</h2> -<p>Rsyslog has a modular design. Consequently, there is a growing +<a href="rsyslog_conf.html">Back to rsyslog.conf manual</a> +<h1>Modules</h1> +<p>Rsyslog has a modular design. This enables functionality to be +dynamically loaded from modules, which may also be written by any +third party. Rsyslog itself offers all non-core functionality as +modules. Consequently, there is a growing number of modules. Here is the entry point to their documentation and what they do (list is currently not complete)</p> +<p>Please note that each module provides configuration +directives, which are NOT necessarily being listed below. Also +remember, that a modules configuration directive (and functionality) is +only available if it has been loaded (using $ModLoad).</p> +<p>It is relatively easy to write a rsyslog module. <b>If none of the provided +modules solve your need, you may consider writing one or have one written +for you by +<a href="http://www.rsyslog.com/professional-services">Adiscon's professional services for rsyslog</a> +</b>(this often is a very cost-effective and efficient way of getting what you need). +<p>There exist different classes of loadable modules: +<ul> +<li><a href="rsyslog_conf_modules.html#im">Input Modules</a> +<li><a href="rsyslog_conf_modules.html#om">Output Modules</a> +<li><a href="rsyslog_conf_modules.html#pm">Parser Modules</a> +<li><a href="rsyslog_conf_modules.html#mm">Message Modification Modules</a> +<li><a href="rsyslog_conf_modules.html#lm">Library Modules</a> +</ul> + +<a name"im"></a><h2>Input Modules</h2> +<p>Input modules are used to gather messages from various sources. They interface +to message generators. +<ul> +<li><a href="imfile.html">imfile</a> - input module for text files</li> +<li><a href="imrelp.html">imrelp</a> - RELP input module</li> +<li>imudp - udp syslog message input</li> +<li><a href="imtcp.html">imtcp</a> - input plugin for plain tcp syslog</li> +<li><a href="imgssapi.html">imgssapi</a> - input plugin for plain tcp and GSS-enabled syslog</li> +<li>immark - support for mark messages</li> +<li><a href="imklog.html">imklog</a> - kernel logging</li> +<li><a href="imuxsock.html">imuxsock</a> - unix sockets, including the system log socket</li> +<li><a href="im3195.html">im3195</a> - accepts syslog messages via RFC 3195</li> +</ul> + +<a name"om"></a><h2>Output Modules</h2> +<p>Output modules process messages. With them, message formats can be transformed +and messages be transmitted to various different targets. <ul> <li><a href="omsnmp.html">omsnmp</a> - SNMP trap output module</li> <li><a href="omstdout.html">omtdout</a> - stdout output module (mainly a test tool)</li> <li><a href="omrelp.html">omrelp</a> - RELP output module</li> +<li><a href="omruleset.html">omruleset</a> - forward message to another ruleset</li> <li>omgssapi - output module for GSS-enabled syslog</li> <li><a href="ommysql.html">ommysql</a> - output module for MySQL</li> <li>ompgsql - output module for PostgreSQL</li> @@ -20,26 +60,54 @@ SQLLite, Ingres, Oracle, mSQL)</li> <li><a href="ommail.html">ommail</a> - permits rsyslog to alert folks by mail if something important happens</li> <li><a href="omoracle.html">omoracle</a> - output module for Oracle (native OCI interface)</li> -<li><a href="imfile.html">imfile</a> -- input module for text files</li> -<li><a href="imrelp.html">imrelp</a> - RELP -input module</li> -<li>imudp - udp syslog message input</li> -<li><a href="imtcp.html">imtcp</a> - input -plugin for plain tcp syslog</li> -<li><a href="imgssapi.html">imgssapi</a> - -input plugin for plain tcp and GSS-enabled syslog</li> -<li>immark - support for mark messages</li> -<li><a href="imklog.html">imklog</a> - kernel logging</li> -<li><a href="imuxsock.html">imuxsock</a> - -unix sockets, including the system log socket</li> -<li><a href="im3195.html">im3195</a> - -accepts syslog messages via RFC 3195</li> +<li><a href="omudpspoof.html">omudpspoof</a> - output module sending UDP syslog messages with a spoofed address</li> </ul> -<p>Please note that each module provides configuration -directives, which are NOT necessarily being listed below. Also -remember, that a modules configuration directive (and functionality) is -only available if it has been loaded (using $ModLoad).</p> + +<a name="pm"></a><h2>Parser Modules</h2> +<p>Parser modules are used to parse message content, once the message has been +received. They can be used to process custom message formats or invalidly formatted +messages. For details, please see the <a href="messageparser.html">rsyslog +message parser documentation</a>. +<p>The current modules are currently provided as part of rsyslog: +<ul> +<li>pmrfc5424 - parses RFC5424-formatted messages (the new syslog standard) +<li>pmrfc3164 - the traditional/legacy syslog parser +</ul> + +<a name="mm"></a><h2>Message Modification Modules</h2> +<p>Message modification modules are used to change the content of messages being processed. +They can be implemented using either the output module or the parser module interface. +From the rsyslog core's point of view, they actually are output or parser modules, it is their +implementation that makes them special. +<p>Currently, there do not exist any such modules, but could be written with +the methods the engine provides. They could be used, for example, to: +<ul> +<li>anonymize message content +<li>add dynamically computed content to message (fields) +</ul> + +<a name="lm"></a><h2>Library Modules</h2> +<p>Library modules provide dynamically loadable functionality for parts of rsyslog, +most often for other loadable modules. They can not be user-configured and are loaded +automatically by some components. They are just mentioned so that error messages that +point to library moduls can be understood. No module list is provided. + +<h2>Where are the modules integrated into the Message Flow?</h2> +<p>Depending on their module type, modules may access and/or modify messages at +various stages during rsyslog's processing. Note that only the "core type" (e.g. input, +output) but not any type derived from it (message modification module) specifies when +a module is called. +<p>The simplified workflow is as follows: +<p align="center"> +<img src="module_workflow.png" alt"rsyslog: loadable modules and message flow"> +<p>As can be seen, messages are received by input modules, then passed to one or many +parser modules, which generate the in-memory representation of the message and may +also modify the message itself. The, the internal representation is passed to +output modules, which may output a message and (with the interfaces newly introduced +in v5) may also modify messageo object content. +<p>Note that the actual flow is much more complex and depends a lot on queue and +filter settings. This graphic above is a high-level message flow diagram. + <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> diff --git a/doc/rsyslog_ng_comparison.html b/doc/rsyslog_ng_comparison.html index 8e121a8d..7d12a4a7 100644 --- a/doc/rsyslog_ng_comparison.html +++ b/doc/rsyslog_ng_comparison.html @@ -5,6 +5,10 @@ <h1>rsyslog vs. syslog-ng</h1> <p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> (2008-05-06)</i></small></p> +<p><i>Warning</i>: this comparison is a little outdated, take it with a grain +of salt and be sure to check the links at the bottom (both syslog-ng as well as +rsyslog features are missing, but our priority is on creating great software not +continously updating this comparison ;)). <p>We have often been asked about a comparison sheet between rsyslog and syslog-ng. Unfortunately, I do not know much about syslog-ng, I did not even use it once. Also, there seems to be no @@ -81,9 +85,10 @@ optional input</td> </tr> <tr> <td valign="top">Windows Event Log</td> -<td valign="top">via <a href="http://www.eventreporter.com">EventReporter</a> +<td valign="top">via a Windows event logging software such as +<a href="http://www.eventreporter.com">EventReporter</a> or <a href="http://www.mwagent.com">MonitorWare Agent</a> -(both commercial software)</td> +(both commercial software, both fund rsyslog development)</td> <td valign="top">via separate Windows agent, paid edition only</td> </tr> diff --git a/doc/rsyslog_pgsql.html b/doc/rsyslog_pgsql.html new file mode 100644 index 00000000..dcb9dc3a --- /dev/null +++ b/doc/rsyslog_pgsql.html @@ -0,0 +1,336 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> +<HTML> +<HEAD> + <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8"> + <TITLE></TITLE> + <META NAME="GENERATOR" CONTENT="OpenOffice.org 3.1 (Unix)"> + <META NAME="AUTHOR" CONTENT="Marc Schiffbauer"> + <META NAME="CREATED" CONTENT="20100129;15054500"> + <META NAME="CHANGEDBY" CONTENT="Marc Schiffbauer"> + <META NAME="CHANGED" CONTENT="20100129;16035000"> + <META NAME="Info 1" CONTENT=""> + <META NAME="Info 2" CONTENT=""> + <META NAME="Info 3" CONTENT=""> + <META NAME="Info 4" CONTENT=""> + <STYLE TYPE="text/css"> + <!-- + @page { size: 8.27in 11.69in; margin: 0.79in } + P { margin-bottom: 0.08in } + P.western { font-family: "Arial", sans-serif } + H1 { margin-bottom: 0.08in } + H1.western { font-family: "Times New Roman", serif } + H1.cjk { font-family: "DejaVu Sans" } + H1.ctl { font-family: "DejaVu Sans" } + H2 { margin-bottom: 0.08in } + H2.western { font-family: "Times New Roman", serif } + BLOCKQUOTE.western { font-family: "Arial", sans-serif } + H3 { margin-bottom: 0.08in } + H3.western { font-family: "Times New Roman", serif } + A:link { so-language: zxx } + --> + </STYLE> +</HEAD> +<BODY LANG="de-DE" DIR="LTR"> +<H1 CLASS="western"><SPAN LANG="en-US">Writing </SPAN>syslog messages +to MySQL, PostgreSQL or any other supported Database</H1> +<P CLASS="western"><FONT SIZE=2><I>Written by </I></FONT><A HREF="http://www.adiscon.com/en/people/rainer-gerhards.php"><FONT SIZE=2><I>Rainer +Gerhards</I></FONT></A><FONT SIZE=2><I> with some additions by Marc +Schiffbauer (2008-02-28)</I></FONT></P> +<H2 CLASS="western">Abstract</H2> +<P CLASS="western"><SPAN LANG="en-US"><I><B>In this paper, I describe +how to write </B></I></SPAN><A HREF="http://www.monitorware.com/en/topics/syslog/">syslog</A><SPAN LANG="en-US"><I><B> +messages to a </B></I></SPAN><A HREF="http://www.mysql.com/">MySQL</A><SPAN LANG="en-US"><I><B> +or </B></I></SPAN><A HREF="http://www.postgresql.org/">PostgreSQL</A><SPAN LANG="en-US"><I><B> +database.</B></I></SPAN><SPAN LANG="en-US"><I> Having syslog messages +in a database is often handy, especially when you intend to set up a +front-end for viewing them. This paper describes an approach with +</I></SPAN><A HREF="http://www.rsyslog.com/">rsyslogd</A><SPAN LANG="en-US"><I>, +an alternative enhanced syslog daemon natively supporting MySQL and +PostgreSQL. I describe the components needed to be installed and how +to configure them. Please note that as of this writing, rsyslog +supports a variety of databases. While this guide is still MySQL- and +PostgreSQL-focused, you can probably use it together with other ones +too. You just need to modify a few settings.</I></SPAN></P> +<H2 CLASS="western">Background</H2> +<P LANG="en-US" CLASS="western">In many cases, syslog data is simply +written to text files. This approach has some advantages, most +notably it is very fast and efficient. However, data stored in text +files is not readily accessible for real-time viewing and analysis. +To do that, the messages need to be in a database. There are various +ways to store syslog messages in a database. For example, some have +the syslogd write text files which are later feed via a separate +script into the database. Others have written scripts taking the data +(via a pipe) from a non-database-aware syslogd and store them as they +appear. Some others use database-aware syslogds and make them write +the data directly to the database. In this paper, I use that "direct +write" approach. I think it is superior, because the syslogd +itself knows the status of the database connection and thus can +handle it intelligently (well ... hopefully ;)). I use rsyslogd to +acomplish this, simply because I have initiated the rsyslog project +with database-awareness as one goal.</P> +<P CLASS="western"><SPAN LANG="en-US"><B>One word of caution:</B></SPAN><SPAN LANG="en-US"> +while message storage in the database provides an excellent +foundation for interactive analysis, it comes at a cost. Database i/o +is considerably slower than text file i/o. As such, directly writing +to the database makes sense only if your message volume is low enough +to allow a) the syslogd, b) the network, and c) the database server +to catch up with it. Some time ago, I have written a paper on +</SPAN><A HREF="http://www.monitorware.com/Common/en/Articles/performance-optimizing-syslog-server.php">optimizing +syslog server performance</A><SPAN LANG="en-US">. While this paper +talks about Window-based solutions, the ideas in it are generic +enough to apply here, too. So it might be worth reading if you +anticipate medium high to high traffic. If you anticipate really high +traffic (or very large traffic spikes), you should seriously consider +forgetting about direct database writes - in my opinion, such a +situation needs either a very specialized system or a different +approach (the text-file-to-database approach might work better for +you in this case). </SPAN> +</P> +<H2 CLASS="western">Overall System Setup</H2> +<P CLASS="western"><SPAN LANG="en-US">In this paper, I concentrate on +the server side. If you are thinking about interactive syslog message +review, you probably want to centralize syslog. In such a scenario, +you have multiple machines (the so-called clients) send their data to +a central machine (called server in this context). While I expect +such a setup to be typical when you are interested in storing +messages in the database, I do not describe how to set it up. This is +beyond the scope of this paper. If you search a little, you will +probably find many good descriptions on </SPAN><SPAN LANG="en-US">how +to centralize syslog. If you do that, it might be a good idea to do +it securely, so you might also be interested in my paper on +</SPAN><A HREF="http://www.rsyslog.com/doc-rsyslog_stunnel.html">ssl-encrypting +syslog message transfer</A><SPAN LANG="en-US">.</SPAN></P> +<P LANG="en-US" CLASS="western">No matter how the messages arrive at +the server, their processing is always the same. So you can use this +paper in combination with any description for centralized syslog +reporting.</P> +<P CLASS="western"><SPAN LANG="en-US">As I already said, I use +rsyslogd on the server. It has intrinsic support for talking to the +supported databases. For obvious reasons, we also need an instance of +MySQL or PostgreSQL running. To keep us focused, the setup of the +database itself is also beyond the scope of this paper. I assume that +you have successfully installed the database and also have a +front-end at hand to work with it (for example, </SPAN><A HREF="http://www.phpmyadmin.net/">phpMyAdmin</A><SPAN LANG="en-US"> +or </SPAN><A HREF="http://phppgadmin.sourceforge.net/">phpPgAdmin</A><SPAN LANG="en-US">. +Please make sure that this is installed, actually working and you +have a basic understanding of how to handle it.</SPAN></P> +<H2 CLASS="western">Setting up the system</H2> +<P CLASS="western"><SPAN LANG="en-US">You need to download and +install rsyslogd first. Obtain it from the </SPAN><A HREF="http://www.rsyslog.com/">rsyslog +site</A><SPAN LANG="en-US">. Make sure that you disable stock +syslogd, otherwise you will experience some difficulties. On some +distributions (Fedora 8 and above, for example), rsyslog may +already by the default syslogd, in which case you obviously do not +need to do anything specific. For many others, there are prebuild +packages available. If you use either, please make sure that you have +the required database plugins for your database available. It usually +is a separate package and typically </SPAN><SPAN LANG="en-US"><B>not</B></SPAN><SPAN LANG="en-US"> +installed by default.</SPAN></P> +<P CLASS="western"><SPAN LANG="en-US">It is important to understand +how rsyslogd talks to the database. In rsyslogd, there is the concept +of "templates". Basically, a template is a string that +includes some replacement characters, which are called "properties" +in rsyslog. Properties are accessed via the "</SPAN><A HREF="http://www.rsyslog.com/doc-property_replacer.html">Property +Replacer</A><SPAN LANG="en-US">". Simply said, you access +properties by including their name between percent signs inside the +template. For example, if the syslog message is "Test", the +template "%msg%" would be expanded to "Test". +Rsyslogd supports sending template text as a SQL statement to the +database. As such, the template must be a valid SQL statement. There +is no limit in what the statement might be, but there are some +obvious and not so obvious choices. For example, a template "drop +table xxx" is possible, but does not make an awful lot of sense. +In practice, you will always use an "insert" statement +inside the template.</SPAN></P> +<P LANG="en-US" CLASS="western">An example: if you would just like to +store the msg part of the full syslog message, you have probably +created a table "syslog" with a single column "message". +In such a case, a good template would be "insert into +syslog(message) values ('%msg%')". With the example above, that +would be expanded to "insert into syslog(message) +values('Test')". This expanded string is then sent to the +database. It's that easy, no special magic. The only thing you must +ensure is that your template expands to a proper SQL statement and +that this statement matches your database design.</P> +<P CLASS="western"><SPAN LANG="en-US">Does that mean you need to +create database schema yourself and also must fully understand +rsyslogd's properties? No, that's not needed. Because we anticipated +that folks are probably more interested in getting things going +instead of designing them from scratch. So we have provided a default +schema as well as build-in support for it. This schema also offers an +additional benefit: rsyslog is part of </SPAN><A HREF="http://www.adiscon.com/en/">Adiscon</A><SPAN LANG="en-US">'s +</SPAN><A HREF="http://www.monitorware.com/en/">MonitorWare product +line</A><SPAN LANG="en-US"> (which includes open source and closed +source members). All of these tools share the same default schema and +know how to operate on it. For this reason, the default schema is +also called the "MonitorWare Schema". If you use it, you +can simply add </SPAN><A HREF="http://www.phplogcon.org/">phpLogCon, +a GPLed syslog web interface</A><SPAN LANG="en-US">, to your system +and have instant interactive access to your database. So there are +some benefits in using the provided schema.</SPAN></P> +<P LANG="en-US" CLASS="western">The schema definition is contained in +the file "createDB.sql". It comes with the rsyslog package +and one can be found for each supported database type (in the plugins +directory). Review it to check that the database name is acceptable +for you. Be sure to leave the table and field names unmodified, +because otherwise you need to customize rsyslogd's default sql +template, which we do not do in this paper. Then, run the script with +your favorite SQL client. Double-check that the table was +successfully created.</P> +<P LANG="en-US" CLASS="western">It is important to note that the +correct database encoding must be used so that the database will +accept strings independend of the string encoding. This is an +important part because it can not be guarantied that all syslog +messages will have a defined character encoding. This is especially +true if the rsyslog-Server will collect messages from different +clients and different products. +</P> +<P LANG="en-US" CLASS="western">For example PostgreSQL may refuse to +accept messages if you would set the database encoding to “UTF8†+while a client is sending invalid byte sequences for that encoding. +</P> +<P LANG="en-US" CLASS="western">Database support in rsyslog is +integrated via loadable plugin modules. To use the database +functionality, the database plugin must be enabled in the config file +BEFORE the first database table action is used. This is done by +placing the</P> +<BLOCKQUOTE CLASS="western"><CODE>$ModLoad ommysql</CODE></BLOCKQUOTE> +<P CLASS="western">directive at the begining of /etc/rsyslog.conf for +MySQL and</P> +<BLOCKQUOTE CLASS="western"><CODE>$ModLoad ompgsql</CODE></BLOCKQUOTE> +<P CLASS="western"><CODE><FONT FACE="Arial, sans-serif">for +PostgreSQL.</FONT></CODE></P> +<P LANG="en-US" CLASS="western"><FONT FACE="Arial, sans-serif">For +other databases, use their plugin name (e.g. omoracle).</FONT></P> +<P CLASS="western">Next, we need to tell rsyslogd to write data to +the database. As we use the default schema, we do NOT need to define +a template for this. We can use the hardcoded one (rsyslogd handles +the proper template linking). So all we need to do e.g. for MySQL is +add a simple selector line to /etc/rsyslog.conf:</P> +<BLOCKQUOTE CLASS="western"><CODE>*.* +:ommysql:database-server,database-name,database-userid,database-password</CODE></BLOCKQUOTE> +<P CLASS="western">Again, other databases have other selector names, +e.g. ":ompgsql:" instead of ":ommysql:". See the +output plugin's documentation for details.</P> +<P LANG="en-US" CLASS="western">In many cases, the database will run +on the local machine. In this case, you can simply use "127.0.0.1" +for <I>database-server</I>. This can be especially advisable, if you +do not need to expose the database to any process outside of the +local machine. In this case, you can simply bind it to 127.0.0.1, +which provides a quite secure setup. Of course, rsyslog also supports +remote database instances. In that case, use the remote server name +(e.g. mydb.example.com) or IP-address. The <I>database-name</I> by +default is "Syslog". If you have modified the default, use +your name here. <I>Database-userid</I> and <I>-password</I> are the +credentials used to connect to the database. As they are stored in +clear text in rsyslog.conf, that user should have only the least +possible privileges. It is sufficient to grant it INSERT privileges +to the systemevents table, only. As a side note, it is strongly +advisable to make the rsyslog.conf file readable by root only - if +you make it world-readable, everybody could obtain the password (and +eventually other vital information from it). In our example, let's +assume you have created a database user named "syslogwriter" +with a password of "topsecret" (just to say it bluntly: +such a password is NOT a good idea...). If your database is on the +local machine, your rsyslog.conf line might look like in this sample:</P> +<BLOCKQUOTE CLASS="western"><CODE>*.* +:ommysql:127.0.0.1,Syslog,syslogwriter,topsecret</CODE></BLOCKQUOTE> +<P CLASS="western">Save rsyslog.conf, restart rsyslogd - and you +should see syslog messages being stored in the "systemevents" +table!</P> +<P LANG="en-US" CLASS="western">The example line stores every message +to the database. Especially if you have a high traffic volume, you +will probably limit the amount of messages being logged. This is easy +to accomplish: the "write database" action is just a +regular selector line. As such, you can apply normal selector-line +filtering. If, for example, you are only interested in messages from +the mail subsystem, you can use the following selector line:</P> +<BLOCKQUOTE CLASS="western"><CODE>mail.* :ommysql:127.0.0.1,syslog,syslogwriter,topsecret</CODE></BLOCKQUOTE> +<P CLASS="western">Review the <A HREF="http://www.rsyslog.com/doc-rsyslog_conf.html">rsyslog.conf</A> +documentation for details on selector lines and their filtering.</P> +<P CLASS="western"><SPAN LANG="en-US"><B>You have now completed +everything necessary to store syslog messages to the a database.</B></SPAN><SPAN LANG="en-US"> +If you would like to try out a front-end, you might want to look at +</SPAN><A HREF="http://www.phplogcon.org/">phpLogCon</A><SPAN LANG="en-US">, +which displays syslog data in a browser. As of this writing, +phpLogCon is not yet a powerful tool, but it's open source, so it +might be a starting point for your own solution.</SPAN></P> +<H2 CLASS="western">On Reliability...</H2> +<P LANG="en-US" CLASS="western">Rsyslogd writes syslog messages +directly to the database. This implies that the database must be +available at the time of message arrival. If the database is offline, +no space is left or something else goes wrong - rsyslogd can not +write the database record. If rsyslogd is unable to store a message, +it performs one retry. This is helpful if the database server was +restarted. In this case, the previous connection was broken but a +reconnect immediately succeeds. However, if the database is down for +an extended period of time, an immediate retry does not help.</P> +<P CLASS="western"><SPAN LANG="en-US">Message loss in this scenario +can easily be prevented with rsyslog. All you need to do is run the +database writer in queued mode. This is now described in a generic +way and I do not intend to duplicate it here. So please be sure to +read "</SPAN><A HREF="http://www.rsyslog.com/doc-rsyslog_high_database_rate.html">Handling +a massive syslog database insert rate with Rsyslog</A><SPAN LANG="en-US">", +which describes the scenario and also includes configuration +examples.</SPAN></P> +<H2 CLASS="western">Conclusion</H2> +<P LANG="en-US" CLASS="western">With minimal effort, you can use +rsyslogd to write syslog messages to a database. You can even make it +absolutely fail-safe and protect it against database server downtime. +Once the messages are arrived there, you can interactively review and +analyze them. In practice, the messages are also stored in text files +for longer-term archival and the databases are cleared out after some +time (to avoid becoming too slow). If you expect an extremely high +syslog message volume, storing it in real-time to the database may +outperform your database server. In such cases, either filter out +some messages or used queued mode (which in general is recommended +with databases).</P> +<P LANG="en-US" CLASS="western">The method outlined in this paper +provides an easy to setup and maintain solution for most use cases.</P> +<H3 CLASS="western">Feedback Requested</H3> +<P CLASS="western">I would appreciate feedback on this paper. If you +have additional ideas, comments or find bugs, please <A HREF="mailto:rgerhards@adiscon.com">let +me know</A>.</P> +<H2 CLASS="western">References and Additional Material</H2> +<UL> + <LI><P CLASS="western" STYLE="margin-bottom: 0in"><A HREF="http://www.rsyslog.com/">www.rsyslog.com</A> + - the rsyslog site + </P> + <LI><P CLASS="western"><A HREF="http://www.monitorware.com/Common/en/Articles/performance-optimizing-syslog-server.php">Paper + on Syslog Server Optimization</A> + </P> +</UL> +<H2 CLASS="western">Revision History</H2> +<UL> + <LI><P CLASS="western" STYLE="margin-bottom: 0in">2005-08-02 * + <A HREF="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer + Gerhards</A> * initial version created + </P> + <LI><P CLASS="western" STYLE="margin-bottom: 0in">2005-08-03 * + <A HREF="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer + Gerhards</A> * added references to demo site + </P> + <LI><P CLASS="western" STYLE="margin-bottom: 0in">2007-06-13 * + <A HREF="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer + Gerhards</A> * removed demo site - was torn down because too + expensive for usage count + </P> + <LI><P CLASS="western" STYLE="margin-bottom: 0in">2008-02-21 * + <A HREF="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer + Gerhards</A> * updated reliability section, can now be done with + on-demand disk queues</P> + <LI><P CLASS="western">2008-02-28 * <A HREF="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer + Gerhards</A> * added info on other databases, updated syntax to more + recent one + </P> + <LI><P CLASS="western">2010-01-29 * Marc Schiffbauer * added some + PostgreSQL stuff, made wording more database generic, fixed some + typos</P> +</UL> +<H2 CLASS="western">Copyright</H2> +<P CLASS="western">Copyright (c) 2005-2010 <A HREF="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer +Gerhards</A>, Marc Schiffbauer and <A HREF="http://www.adiscon.com/en/">Adiscon</A>.</P> +<P CLASS="western"><BR><BR> +</P> +</BODY> +</HTML>
\ No newline at end of file diff --git a/doc/rsyslog_php_syslog_ng.html b/doc/rsyslog_php_syslog_ng.html index 9e722755..ed4d72fc 100644 --- a/doc/rsyslog_php_syslog_ng.html +++ b/doc/rsyslog_php_syslog_ng.html @@ -7,8 +7,10 @@ <P><small><i>Written by <a href="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer Gerhards</a> (2005-08-04)</i></small></P> -<p><b>Note: it has been reported that this guide is somewhat outdated. Please -use with care. </b></p> +<p>Note: it has been reported that this guide is somewhat outdated. Please +use with care. Also, please note that <b>rsyslog's "native" web frontend is +<a href="http://www.phplogcon.org">phpLogCon</a></b>, which provides best integration +and a lot of extra functionality.</p> <h2>Abstract</h2> <p><i><b>In this paper, I describe how to use <a href="http://www.vermeer.org/projects/php-syslog-ng">php-syslog-ng</a> with @@ -107,7 +109,7 @@ server machine, "syslog" is the database name (default from the schema and "pass" are the logon credentials. Use a user with low privileges, insert into the logs table is sufficient. "syslog-ng" is the template name and tells rsyslogd to use the SQL statement shown above.</p> -<p>Once you have made the changes, all you need to do is reload (or HUP) +<p>Once you have made the changes, all you need to do is restart rsyslogd. Then, you should see syslog messages flow into your database - and show up in php-syslog-ng.</p> <h2>Conclusion</h2> @@ -116,11 +118,11 @@ those unfamiliar with syslog-ng, this configuration is probably easier to set up then switching to syslog-ng. For existing rsyslogd users, php-syslog-ng might be a nice add-on to their logging infrastructure.</P> <P>Please note that the <a href="http://www.monitorware.com/en/">MonitorWare family</a> (to which rsyslog belongs) also -offers a web-interface: <a href="http://www.phplogcon.org/">phpLogCon</a>. At the time of this writing, phpLogCon's code -is by far not as clean as I would like it to be. Also the user-interface is -definitely not as intutive as pp-syslog-ng. From a functionality point of view, -however, I think it already is a bit ahead. So you might -consider using it. I have set up a <a href="http://demo.rsyslog.com/">demo server</a>., +offers a web-interface: <a href="http://www.phplogcon.org/">phpLogCon</a>. +From my point of view, obviously, <b>phpLogCon is the more natural choice for a web interface +to be used together with rsyslog</b>. It also offers superb functionality and provides, +for example,native display of Windows event log entries. +I have set up a <a href="http://demo.phplogcon.org/">demo server</a>., You can have a peek at it without installing anything.</P> <h2>Feedback Requested</h2> @@ -148,4 +150,4 @@ no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at <a href="http://www.gnu.org/copyleft/fdl.html"> http://www.gnu.org/copyleft/fdl.html</a>.</p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsyslog_queue_pointers.jpeg b/doc/rsyslog_queue_pointers.jpeg Binary files differnew file mode 100644 index 00000000..809dd446 --- /dev/null +++ b/doc/rsyslog_queue_pointers.jpeg diff --git a/doc/rsyslog_queue_pointers2.jpeg b/doc/rsyslog_queue_pointers2.jpeg Binary files differnew file mode 100644 index 00000000..2ad60113 --- /dev/null +++ b/doc/rsyslog_queue_pointers2.jpeg diff --git a/doc/rsyslog_recording_pri.html b/doc/rsyslog_recording_pri.html index 1dcf00c7..cf11e3e5 100644 --- a/doc/rsyslog_recording_pri.html +++ b/doc/rsyslog_recording_pri.html @@ -62,7 +62,7 @@ semicolon:</p> <p>That's all you need to do. There is one common pitfall: you need to define the template before you use it in a selector line. Otherwise, you will receive an error.</p> -<p>Once you have applied the changes, you need to restart or HUP rsyslogd. It +<p>Once you have applied the changes, you need to restart rsyslogd. It will then pick the new configuration.</p> <h2>What if I do not want rsyslogd to be the standard syslogd?</h2> <p>If you do not want to switch to rsyslog, you can still use it as a setup aid. diff --git a/doc/rsyslog_secure_tls.html b/doc/rsyslog_secure_tls.html index be2811f4..b15e5a4e 100644 --- a/doc/rsyslog_secure_tls.html +++ b/doc/rsyslog_secure_tls.html @@ -51,7 +51,7 @@ google_ad_height = 125; src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> </script> </span> -I private keys have become known to third parties, the system does not provide +If private keys have become known to third parties, the system does not provide any security at all. Also, our solution bases on X.509 certificates and a (very limited) chain of trust. We have one instance (the CA) that issues all machine certificates. The machine certificate indentifies a particular machine. hile in diff --git a/doc/rsyslog_tls.html b/doc/rsyslog_tls.html index e37d26a7..bb312c77 100644 --- a/doc/rsyslog_tls.html +++ b/doc/rsyslog_tls.html @@ -128,7 +128,7 @@ This is all you need to do. You can use the rest of your rsyslog.conf together with this configuration. The way messages are received does not interfer with any other option, so you are able to do anything else you like without any restrictions. -<p>Restart (or HUP) rsyslogd. The server should now be fully +<p>Restart rsyslogd. The server should now be fully operational.</p> <h3>Client Setup</h3> <p>The client setup is equally simple. You need less diff --git a/doc/src/classes.dia b/doc/src/classes.dia Binary files differindex 70e91566..8cfcbd0c 100644 --- a/doc/src/classes.dia +++ b/doc/src/classes.dia diff --git a/doc/src/module_workflow.dia b/doc/src/module_workflow.dia Binary files differnew file mode 100644 index 00000000..178571f4 --- /dev/null +++ b/doc/src/module_workflow.dia diff --git a/doc/src/rfc5424layers.dia b/doc/src/rfc5424layers.dia Binary files differnew file mode 100644 index 00000000..300b7796 --- /dev/null +++ b/doc/src/rfc5424layers.dia diff --git a/doc/src/rsyslog_pgsql.odt b/doc/src/rsyslog_pgsql.odt Binary files differnew file mode 100644 index 00000000..5034c5fb --- /dev/null +++ b/doc/src/rsyslog_pgsql.odt diff --git a/doc/src/rsyslog_queue_pointers.dia b/doc/src/rsyslog_queue_pointers.dia Binary files differnew file mode 100644 index 00000000..2ad4cacb --- /dev/null +++ b/doc/src/rsyslog_queue_pointers.dia diff --git a/doc/src/rsyslog_queue_pointers2.dia b/doc/src/rsyslog_queue_pointers2.dia Binary files differnew file mode 100644 index 00000000..6a35c664 --- /dev/null +++ b/doc/src/rsyslog_queue_pointers2.dia diff --git a/doc/src/tls.dia b/doc/src/tls.dia Binary files differindex 77e5d185..d7c9811d 100644 --- a/doc/src/tls.dia +++ b/doc/src/tls.dia diff --git a/doc/status.html b/doc/status.html index c8608f73..801decfc 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,23 +2,38 @@ <html><head><title>rsyslog status page</title></head> <body> <h2>rsyslog status page</h2> -<p>This page reflects the status as of 2009-04-21.</p> +<p>This page reflects the status as of 2010-02-10.</p> <h2>Current Releases</h2> -<p><b>development:</b> 4.3.0 [2009-04-17] - -<a href="http://www.rsyslog.com/Article366.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-156.phtml">download</a> +<p><b>development:</b> 5.5.2 [2010-02-05] - +<a href="http://www.rsyslog.com/Article439.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-193.phtml">download</a> +<!-- not at the moment! +<br> +<b>v4 development:</b> 4.5.1 [2009-07-15] - +<a href="http://www.rsyslog.com/Article388.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-167.phtml">download</a></p> +--> -<br><b>beta:</b> 3.21.11 [2009-04-03] - -<a href="http://www.rsyslog.com/Article358.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-152.phtml">download</a></p> +<p><b>beta:</b> 5.3.7 [2010-01-27] - +<a href="http://www.rsyslog.com/Article437.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-192.phtml">download</a></p> -<p><b>v3 stable:</b> 3.22.0 [2009-04-21] - <a href="http://www.rsyslog.com/Article368.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-157.phtml">download</a> +<p><b>v5 stable:</b> 5.2.0 [2009-11-02] (recommended to use v5-beta instead) - +<a href="http://www.rsyslog.com/Article421.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-184.phtml">download</a> -<br><b>v2 stable:</b> 2.0.7 [2009-04-14] - <a href="http://www.rsyslog.com/Article362.phtml">change log</a> - +<br><b>v4 stable:</b> 4.6.0 [2010-02-24] - +<a href="http://www.rsyslog.com/Article444.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-195.phtml">download</a> + +<br><b>v3 stable:</b> 3.22.1 [2009-07-02] - +<a href="http://www.rsyslog.com/Article381.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-163.phtml">download</a> + +<br>v2 stable: 2.0.7 [2009-04-14] - <a href="http://www.rsyslog.com/Article362.phtml">change log</a> - <a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-154.phtml">download</a> -<br>v0 and v1 are deprecated and no longer supported. If you absolutely do not like to +<br>v0 to v2 are deprecated and no longer supported. If you absolutely do not like to upgrade, you may consider purchasing a <a href="professional_support.html">commercial rsyslog support package</a>. Just let us point out that it is really not a good idea to still run a v0 version. diff --git a/doc/syslog_parsing.html b/doc/syslog_parsing.html index 57da6657..1ccec6f1 100644 --- a/doc/syslog_parsing.html +++ b/doc/syslog_parsing.html @@ -176,6 +176,19 @@ $template, MalfromedMsgFormater,"%timegenerated% %fromhost% %rawmsg:::drop-last- <p>This will make your log much nicer, but not look perfect. Experiment a bit with the available properties and replacer extraction options to fine-tune it to your needs. +<h2>The Ultimate Solution...</h2> +<p>Is available with rsyslog 5.3.4 and above. Here, we can define so-called custom +parsers. These are plugin modules, written in C and adapted to a specific message format +need. The big plus of custom parsers is that they offer excellent performance and unlimited +possibilities - far better than any work-around could do. Custom parsers can be +<a href="rsconf1_rulesetparser.html">bound to specific rule sets</a> +(and thus listening) ports with relative ease. The only con is that they must be written. +However, if you are lucky, a parser for your device may already exist. If not, you can +opt to write it yourself, what is not too hard if you know some C. Alternatively, +Adiscon can program one for you as part of the +<a href="http://www.rsyslog.com/professional-services">rsyslog professional services offering</a>. +In any case, you should seriously consider custom parsers as an alternative if you can not +reconfigure your device to send decent message format. <h2>Wrap-Up</h2> <p>Syslog message format is not sufficiently standardized. There exists a weak "standard" format, which is used by a good number of implementations. However, there @@ -183,14 +196,15 @@ exist many others, including mainstream vendor implementations, which have a (sometimes horribly) different format. Rsyslog tries to deal with anomalies but can not guess right in all instances. If possible, the sender should be configured to submit well-formed messages. If that is not possible, you can work around these -issues with rsyslog's property replacer and template system. +issues with rsyslog's property replacer and template system. Or you can use a suitable +message parser or write one for your needs. <p>I hope this is a useful guide. You may also have a look at the <a href="troubleshoot.html">rsyslog troubleshooting guide</a> for further help and places where to ask questions. <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body></html> diff --git a/doc/tls_cert_server.html b/doc/tls_cert_server.html index 9c68db5d..9c024bc9 100644 --- a/doc/tls_cert_server.html +++ b/doc/tls_cert_server.html @@ -37,6 +37,15 @@ src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> </script> </span> <p><center><img src="tls_cert_100.jpg"></center> +<p><i><font color="red"><b>Important:</b> Keep in mind that the order of configuration directives +is very important in rsyslog. As such, the samples given below do only work if the given +order is preserved.</font> Re-ordering the directives can break configurations and has broken them +in practice. If you intend to re-order them, please be sure that you fully understand how +the configuration language works and, most importantly, which statements form a block together. +Please also note that we understand the the current configuration file format is +ugly. However, there has been more important work in the way of enhancing it. If you would like +to contribute some time to improve the config file language, please let us know. Any help +is appreciated (be it doc or coding work!).</i> <p>Steps to do: <ul> <li>make sure you have a functional CA (<a href="tls_cert_ca.html">Setting up the CA</a>) diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index a8855fd4..16b2754b 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -102,13 +102,63 @@ comes without any guarantees, include no guarantee on confidentiality [aka "we don't want to be sued for work were are not even paid for ;)]. <b>So if you submit debug logs, do so at your sole risk</b>. By submitting them, you accept this policy. +<p><b>Segmentation Faults</b> +<p>Rsyslog has a very rapid development process, complex capabilities and now gradually gets +more and more exposure. While we are happy about this, it also has some bad effects: some +deployment scenarios have probably never been tested and it may be impossible to test +them for the development team because of resources needed. So while we try to avoid this, +you may see a serious problem during deployments in demanding, non-standard, environments +(hopefully not with a stable version, but chances are good you'll run into troubles with +the development versions). +<p>Active support from the user base is very important to help us track down those things. +Most often, serious problems are the result of some memory misadressing. During development, +we routinely use valgrind, a very well and capable memory debugger. This helps us to create +pretty clean code. But valgrind can not detect anything, most importantly not code pathes +that are never executed. So of most use for us is information about aborts and abort locations. +<p>Unforutnately, faults rooted in adressing errors typically show up only later, so the +actual abort location is in an unrelated spot. To help track down the original spot, +<a href="http://www.gnu.org/software/hello/manual/libc/Heap-Consistency-Checking.html">libc +later than 5.4.23 offers support</a> for finding, and possible temporary relief from it, +by means of the MALLOC_CHECK_ environment variable. Setting it to 2 is a useful troubleshooting +aid for us. It will make the program abort as soon as the check routines detect anything +suspicious (unfortunately, this may still not be the root cause, but hopefully closer to it). +Setting it to 0 may even make some problems disappear (but it will NOT fix them!). +With functionality comes cost, and so exporting MALLOC_CHECK_ without need comes at +a performance penalty. However, we strongly recommend adding this instrumentation to your +test environment should you see any serious problems. Chances are good it will help us +interpret a dump better, and thus be able to quicker craft a fix. +<p>In order to get useful information, we need some backtrace of the abort. First, you need +to make sure that a core file is created. Under Fedora, for example, that means you need +to have an "ulimit -c unlimited" in place. +<p>Now let's assume you got a core file (e.g. in /core.1234). So what to do next? Sending a +core file to us is most often pointless - we need to have the exact same system configuration in +order to interpret it correctly. Obviously, chances are extremely slim for this to be. So we would +appreciate if you could extract the most important information. This is done as follows: +<ul> +<li>$gdb /path/to/rsyslogd +<li>$info thread +<li>you'll see a number of threads (in the range 0 to n with n being the highest number). For + <b>each</b> of them, do the following (let's assume that i is the thread number): + <ul> + <li>$ thread i (e.g. thread 0, thread 1, ...) + <li>$bt + </ul> +<li>then you can quit gdb with "$q" +</ul> +<p>Then please send all information that gdb spit out to the development team. It is best to first +ask on the forum or mailing list on how to do that. The developers will keep in contact with you +and, I fear, will probably ask for other things as well ;) +<p>Note that we strive for highest reliability of the engine even in unusual deployment scenarios. +Unfortunately, this is hard to achieve, especially with limited resources. So we are depending on +cooperation from users. This is your chance to make a big contribution to the project without the +need to program or do anything else except get a problem solved ;) <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/v4compatibility.html b/doc/v4compatibility.html new file mode 100644 index 00000000..5d877af1 --- /dev/null +++ b/doc/v4compatibility.html @@ -0,0 +1,77 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Compatibility notes for rsyslog v4</title> +</head> +<body> +<h1>Compatibility Notes for rsyslog v4</h1> +<p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> +(2009-07-15)</i></small></p> +<p>The changes introduced in rsyslog v4 are numerous, but not very intrusive. +This document describes things to keep in mind when moving from v3 to v4. It +does not list enhancements nor does it talk about compatibility concerns introduced +by v3 (for this, see the <a href="v3compatibility.html">rsyslog v3 compatibility notes</a>). +<h2>HUP processing</h2> +<p>With v3 and below, rsyslog used the traditional HUP behaviour. That meant that +all output files are closed and the configuration file is re-read and the new configuration +applied. +<p>With a program as simple and static as sysklogd, this was not much of an issue. The +most important config settings (like udp reception) of a traditional syslogd can not be +modified via the configuration file. So a config file reload only meant setting up a new set of filters. It also didn't account as problem that while doing so messages may be lost - without +any threading and queuing model, a traditional syslogd will potentially always loose +messages, so it is irrelevant if this happens, too, during the short config re-read +phase. +<p>In rsyslog, things are quite different: the program is more or less a framework into +which loadable modules are loaded as needed for a particular configuration. The software +that will acutally be running is taylored via the config file. Thus, a re-read of +the config file requires a full, very heavy restart, because the software acutally +running with the new config can be totally different from what ran with the old config. +<p>Consequently, the traditional HUP is a very heavy operation and may even cause some +data loss because queues must be shut down, listeners stopped and so on. Some of these +operations (depending on their configuration) involve intentional message loss. The operation +also takes up a lot of system resources and needs quite some time (maybe seconds) to be +completed. During this restart period, the syslog subsytem is not fully available. +<p>From the software developer's point of view, the full restart done by a HUP is rather complex, +especially if user-timeout limits set on action completion are taken into consideration (for +those in the know: at the extreme ends this means we need to cancel threads as a last resort, +but than we need to make sure that such cancellation does not happen at points where it +would be fatal for a restart). A regular restart, where the process is actually terminated, is +much less complex, because the operating system does a full cleanup after process termination, +so rsyslogd does not need to take care for exotic cleanup cases and leave that to the OS. +In the end result, restart-type HUPs clutter the code, increase complexity (read: add bugs) +and cost performance. +<p>On the contrary, a HUP is typically needed for log rotation, and the real desire is +to close files. This is a non-disruptive and very lightweigth operation. +<p>Many people have said that they are used to HUP the syslogd to apply configuration +changes. This is true, but it is questionable if that really justifies all the cost that +comes with it. After all, it is the difference between typing +<pre> +$ kill -HUP `cat /var/run/rsyslogd.pid` +</pre> +versus +<pre> +$ /etc/init.d/rsyslog restart +</pre> +Semantically, both is mostly the same thing. The only difference is that with the restart +command rsyslogd can spit config error message to stderr, so that the user is able to see +any problems and fix them. With a HUP, we do not have access to stderr and thus can log +error messages only to their configured destinations; exprience tells that most users +will never find them there. What, by the way, is another strong argument against +restarting rsyslogd by HUPing it. +<p>So a restart via HUP is not strictly necessary +and most other deamons require that a restart command is typed in if a restart is required. +<p>Rsyslog will follow this paradigm in the next versions, resulting in many benefits. In v4, +we provide some support for the old-style semantics. We introduced a setting $HUPisRestart +which may be set to "on" (tradional, heavy operationg) +or "off" (new, lightweight "file close only" operation). +The initial versions had the default set to traditional behavior, but starting with 4.5.1 +we are now using the new behavior as the default. +<p>Most importantly, <b>this may break some scripts</b>, but my sincere belief is that +there are very few scripts that automatically <b>change</b> rsyslog's config and then do a +HUP to reload it. Anyhow, if you have some of these, it may be a good idea to change +them now instead of turning restart-type HUPs on. Other than that, one mainly needs +to change the habit of how to restart rsyslog after a configuration change. +<p><b>Please note that restart-type HUP is depricated and will go away in rsyslog v5.</b> +So it is a good idea to become ready for the new version now and also enjoy some of the +benefits of the "real restart", like the better error-reporting capability. +<p>Note that code complexity reduction (and thus performance improvement) needs the restart-type +HUP code to be removed, so these changes can (and will) only happen in version 5. +</body></html> diff --git a/doc/v5compatibility.html b/doc/v5compatibility.html new file mode 100644 index 00000000..6d60062f --- /dev/null +++ b/doc/v5compatibility.html @@ -0,0 +1,30 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Compatibility notes for rsyslog v5</title> +</head> +<body> +<h1>Compatibility Notes for rsyslog v5</h1> +<p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> +(2009-07-15)</i></small></p> +<p>The changes introduced in rsyslog v5 are numerous, but not very intrusive. +This document describes things to keep in mind when moving from v4 to v5. It +does not list enhancements nor does it talk about compatibility concerns introduced +by earlier versions (for this, see their respective compatibility documents). +<h2>HUP processing</h2> +<p>The $HUPisRestart directive is supported by some early v5 versions, but has been removed +in 5.1.3 and above. That means that restart-type HUP processing is no longer +available. This processing was redundant and had a lot a drawbacks. +For details, please see the +<a href="v4compatibility.html">rsyslog v4 compatibility notes</a> which elaborate +on the reasons and the (few) things you may need to change. +<h2>Queue Worker Thread Shutdown</h2> +<p>Previous rsyslog versions had the capability to "run" on zero queue worker +if no work was required. This was done to save a very limited number of resources. However, +it came at the price of great complexity. In v5, we have decided to let a minium of one +worker run all the time. The additional resource consumption is probably not noticable at +all, however, this enabled us to do some important code cleanups, resulting in faster +and more reliable code (complex code is hard to maintain and error-prone). From the +regular user's point of view, this change should be barely noticable. I am including the +note for expert users, who will notice it in rsyslog debug output and other analysis tools. +So it is no error if each queue in non-direct mode now always runs at least one worker +thread. +</body></html> @@ -51,11 +51,15 @@ #include "obj.h" #include "errmsg.h" #include "gss-misc.h" +#include "debug.h" +#include "glbl.h" +#include "unlimited_select.h" MODULE_TYPE_LIB /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) static void display_status_(char *m, OM_uint32 code, int type) @@ -108,28 +112,38 @@ static int read_all(int fd, char *buf, unsigned int nbyte) { int ret; char *ptr; - fd_set rfds; struct timeval tv; +#ifdef USE_UNLIMITED_SELECT + fd_set *pRfds = malloc(glbl.GetFdSetSize()); +#else + fd_set rfds; + fd_set *pRfds = &rfds; +#endif for (ptr = buf; nbyte; ptr += ret, nbyte -= ret) { - FD_ZERO(&rfds); - FD_SET(fd, &rfds); + FD_ZERO(pRfds); + FD_SET(fd, pRfds); tv.tv_sec = 1; tv.tv_usec = 0; - if ((ret = select(FD_SETSIZE, &rfds, NULL, NULL, &tv)) <= 0 - || !FD_ISSET(fd, &rfds)) + if ((ret = select(FD_SETSIZE, pRfds, NULL, NULL, &tv)) <= 0 + || !FD_ISSET(fd, pRfds)) { + freeFdSet(pRfds); return ret; + } ret = recv(fd, ptr, nbyte, 0); if (ret < 0) { if (errno == EINTR) continue; + freeFdSet(pRfds); return (ret); } else if (ret == 0) { + freeFdSet(pRfds); return (ptr - buf); } } + freeFdSet(pRfds); return (ptr - buf); } @@ -177,7 +191,7 @@ static int recv_token(int s, gss_buffer_t tok) | lenbuf[3]); tok->length = ntohl(len); - tok->value = (char *) malloc(tok->length ? tok->length : 1); + tok->value = (char *) MALLOC(tok->length ? tok->length : 1); if (tok->length && tok->value == NULL) { errmsg.LogError(0, NO_ERRCODE, "Out of memory allocating token data\n"); return -1; @@ -264,6 +278,7 @@ BEGINObjClassExit(gssutil, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END M CODESTARTObjClassExit(gssutil) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); ENDObjClassExit(gssutil) @@ -274,6 +289,7 @@ ENDObjClassExit(gssutil) BEGINAbstractObjClassInit(gssutil, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE class also in END MACRO! */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(gssutil) diff --git a/java/Makefile.am b/java/Makefile.am new file mode 100644 index 00000000..67f5eb43 --- /dev/null +++ b/java/Makefile.am @@ -0,0 +1,35 @@ +# very rough support for compiling the java components of rsyslog +# Some usage notes: you need to use the Sun JDK compiler (jdk-devel) +# with this. At least it didn't work for me with the eclipse compiler. +# There is no real installation support. If you intend to run a program, +# change to the ./java subdirectory and issue +# java -cp . <class> +# e.g.: java -cp . com.rsyslog.gui.diaggui.DiagGUI +# or any equivalent command. +# +# I am very glad to hear suggestions about how to improve this part +# of the build system. -- rgerhards, 2009-08-27 + +javadir = $(top_builddir)/java +JAVAROOT = $(javadir) +# I don't know why CLASSPATH_ENV works this way, but at least it works... +CLASSPATH_ENV = CLASSPATH=$(javadir):$$CLASSPATH + +JAVA_SOURCE_FILES = \ + com/rsyslog/lib/DiagSess.java \ + com/rsyslog/lib/SyslogMessage.java \ + com/rsyslog/lib/SyslogMsgConsumer.java \ + com/rsyslog/lib/SyslogTrafficGenerator.java \ + com/rsyslog/lib/SyslogSender.java \ + com/rsyslog/lib/UDPSyslogSender.java \ + com/rsyslog/diag/DiagTalker.java \ + com/rsyslog/gui/simpServ/simpServ.java \ + com/rsyslog/gui/simpServ/simpServConsumer.java \ + com/rsyslog/gui/msggen/MsgGen.java \ + com/rsyslog/gui/diaggui/Counters.java \ + com/rsyslog/gui/diaggui/DiagGUI.java + + +java_JAVA = $(JAVA_SOURCE_FILES) + +dist_java = $(JAVA_SOURCE_FILES) diff --git a/java/com/rsyslog/diag/DiagTalker.java b/java/com/rsyslog/diag/DiagTalker.java new file mode 100644 index 00000000..c4e77e95 --- /dev/null +++ b/java/com/rsyslog/diag/DiagTalker.java @@ -0,0 +1,70 @@ +/* A yet very simple tool to talk to imdiag. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.diag; +import java.io.*; +import java.net.*; + +public class DiagTalker { + public static void main(String[] args) throws IOException { + + Socket diagSocket = null; + PrintWriter out = null; + BufferedReader in = null; + final String host = "127.0.0.1"; + final int port = 13500; + + try { + diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(0); /* wait for lenghty operations */ + out = new PrintWriter(diagSocket.getOutputStream(), true); + in = new BufferedReader(new InputStreamReader( + diagSocket.getInputStream())); + } catch (UnknownHostException e) { + System.err.println("can not resolve " + host + "!"); + System.exit(1); + } catch (IOException e) { + System.err.println("Couldn't get I/O for " + + "the connection to: " + host + "."); + System.exit(1); + } + + BufferedReader stdIn = new BufferedReader( + new InputStreamReader(System.in)); + String userInput; + + try { + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + } catch (SocketException e) { + System.err.println("We had a socket exception and consider this to be OK: " + + e.getMessage()); + } + + out.close(); + in.close(); + stdIn.close(); + diagSocket.close(); + } +} + diff --git a/java/com/rsyslog/gui/diaggui/Counters.java b/java/com/rsyslog/gui/diaggui/Counters.java new file mode 100644 index 00000000..363bff43 --- /dev/null +++ b/java/com/rsyslog/gui/diaggui/Counters.java @@ -0,0 +1,138 @@ +/* Display some basic rsyslogd counter variables. + * + * Please note that this program requires imdiag to be loaded inside rsyslogd. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.diaggui; +import java.awt.*; +import java.awt.event.*; +import java.io.*; +import java.util.*; + +import com.rsyslog.lib.DiagSess; + +public class Counters extends Frame { + + private TextField MainQItems; + private TextField RefreshInterval; + private Checkbox AutoRefresh; + private DiagSess diagSess; + private Timer timer; + + private void createDiagSess() { + try { + diagSess = new DiagSess("127.0.0.1", 13500); // TODO: values from GUI + diagSess.connect(); + } + catch(IOException e) { + System.out.println("Exception trying to open diag session:\n" + e.toString()); + } + } + + private void createGUI() { + MainQItems = new TextField(); + MainQItems.setColumns(8); + Panel pCenter = new Panel(); + pCenter.setLayout(new FlowLayout()); + pCenter.add(new Label("MainQ Items:")); + pCenter.add(MainQItems); + + RefreshInterval = new TextField(); + RefreshInterval.setColumns(5); + RefreshInterval.setText("100"); + AutoRefresh = new Checkbox("Auto Refresh", false); + AutoRefresh.addItemListener(new ItemListener() { + public void itemStateChanged(ItemEvent e) { + setAutoRefresh(); + } + + }); + Button b = new Button("Refresh now"); + b.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + refreshCounters(); + } + }); + Panel pSouth = new Panel(); + pSouth.setLayout(new FlowLayout()); + pSouth.add(AutoRefresh); + pSouth.add(new Label("Interval (ms):")); + pSouth.add(RefreshInterval); + pSouth.add(b); + + pack(); + setTitle("rsyslogd Counters"); + setLayout(new BorderLayout()); + add(pCenter, BorderLayout.CENTER); + add(pSouth, BorderLayout.SOUTH); + setSize(400,500); + } + + public Counters() { + addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e){ + Counters.this.dispose(); + } + }); + createGUI(); + createDiagSess(); + setAutoRefresh(); + setVisible(true); + } + + + private void startTimer() { + timer = new Timer(); + timer.scheduleAtFixedRate(new TimerTask() { + public void run() { + refreshCounters(); + } + }, 0, 100); + } + + private void stopTimer() { + if(timer != null) { + timer.cancel(); + timer = null; + } + } + + /** set auto-refresh mode. It is either turned on or off, depending on the + * status of the relevant check box. */ + private void setAutoRefresh() { + if(AutoRefresh.getState() == true) { + startTimer(); + } else { + stopTimer(); + } + } + + /** refresh counter display from rsyslogd data. Does a network round-trip. */ + private void refreshCounters() { + try { + String res = diagSess.request("getmainmsgqueuesize"); + MainQItems.setText(res); + } + catch(IOException e) { + System.out.println("Exception during request:\n" + e.toString()); + } + } +} diff --git a/java/com/rsyslog/gui/diaggui/DiagGUI.java b/java/com/rsyslog/gui/diaggui/DiagGUI.java new file mode 100644 index 00000000..1a03299c --- /dev/null +++ b/java/com/rsyslog/gui/diaggui/DiagGUI.java @@ -0,0 +1,77 @@ +/* A yet very simple diagnostic GUI for rsyslog. + * + * Please note that this program requires imdiag to be loaded inside rsyslogd. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.diaggui; +import java.awt.*; +import java.awt.event.*; + +public class DiagGUI extends Frame { + public Counters counterWin; + public static void main(String args[]) { + new DiagGUI(); + } + + /** show counter window. creates it if not already present */ + public void showCounters() { + if(counterWin == null) { + counterWin = new Counters(); + } else { + counterWin.setVisible(true); + counterWin.toFront(); + } + } + + /** initialize the GUI. */ + public DiagGUI(){ + MenuItem item; + MenuBar menuBar = new MenuBar(); + Menu fileMenu = new Menu("File"); + item = new MenuItem("Exit"); + item.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + System.exit(0); + } + }); + fileMenu.add(item); + menuBar.add(fileMenu); + + Menu viewMenu = new Menu("View"); + item = new MenuItem("Counters"); + item.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + showCounters(); + } + }); + viewMenu.add(item); + menuBar.add(viewMenu); + + addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e){ + System.exit(0); + } + }); + setMenuBar(menuBar); + setSize(300,400); + setVisible(true); + } +} diff --git a/java/com/rsyslog/gui/msggen/MsgGen.java b/java/com/rsyslog/gui/msggen/MsgGen.java new file mode 100644 index 00000000..c57027ff --- /dev/null +++ b/java/com/rsyslog/gui/msggen/MsgGen.java @@ -0,0 +1,140 @@ +/* A yet very simple syslog message generator + * + * The purpose of this program is to provide a facility that enables + * to generate complex traffic patterns for testing purposes. It still is + * in its infancy, but hopefully will evolve. + * + * Note that this has been created as a stand-alone application because it + * was considered useful to have it as a separate program. But it should still + * be possible to call its class from any other program, specifically the debug + * GUI. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.msggen; +import com.rsyslog.lib.*; +import java.awt.*; +import java.awt.event.*; +import javax.swing.*; + +public class MsgGen extends Frame { + private TextField target; + private TextField message; + private TextField nummsgs; + private TextField numthrds; + + public static void main(String args[]) { + new MsgGen(); + } + + /** creates the menu bar INCLUDING all menu handlers */ + private void createMenu() { + MenuItem item; + MenuBar menuBar = new MenuBar(); + Menu fileMenu = new Menu("File"); + item = new MenuItem("Exit"); + item.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + System.exit(0); + } + }); + fileMenu.add(item); + menuBar.add(fileMenu); + + addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e){ + System.exit(0); + } + }); + setMenuBar(menuBar); + } + + /** creates the main GUI */ + private void createGUI() { + //target = new TextField("127.0.0.1", 32); + target = new TextField("172.19.3.3", 32); + message = new TextField(80); + //message.setText("<161>Test malformed"); + message.setText("<5>iaalog[171652]: AIB|dcu|2009/08/12 14:48:43|mfa challenge|NNNNNNN|XX.XX.XX.XX"); + nummsgs = new TextField("1000", 8); + numthrds = new TextField("10", 5); + Panel pCenter = new Panel(); + + Panel pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Target Host:")); + pnl.add(target); + pCenter.add(pnl); + + pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Number of Msgs:")); + pnl.add(nummsgs); + pCenter.add(pnl); + + pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Msg:")); + pnl.add(message); + pCenter.add(pnl); + + Panel pSouth = new Panel(); + pSouth.setLayout(new FlowLayout()); + + pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Number of Threads:")); + pnl.add(numthrds); + pSouth.add(pnl); + + Button b = new Button("Start Test"); + b.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + performTest(); + } + }); + pSouth.add(b); + + pack(); + setTitle("Syslog Message Generator"); + setLayout(new BorderLayout()); + add(pCenter, BorderLayout.CENTER); + add(pSouth, BorderLayout.SOUTH); + setSize(800,400); + } + + /** perform the test, a potentially complex operation */ + private void performTest() { + for(short i = 0 ; i < Integer.parseInt(numthrds.getText()) ; ++ i) { + SyslogTrafficGenerator gen = + new SyslogTrafficGenerator(target.getText(), message.getText(), + Long.parseLong(nummsgs.getText())); + gen.start(); + } + } + + + /** initialize the GUI. */ + public MsgGen(){ + createMenu(); + createGUI(); + setVisible(true); + } +} diff --git a/java/com/rsyslog/gui/simpServ/simpServ.java b/java/com/rsyslog/gui/simpServ/simpServ.java new file mode 100644 index 00000000..2a83dad0 --- /dev/null +++ b/java/com/rsyslog/gui/simpServ/simpServ.java @@ -0,0 +1,45 @@ +/** + * Implementation of a tcp-based syslog server. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +package com.rsyslog.gui.simpServ; +import com.rsyslog.lib.*; +//import com.rsyslog.gui.*; + +public class simpServ { + + public static void main(String args[]) { + try { + simpServConsumer cons = new simpServConsumer(); + System.out.println("Starting server on port " + args[0] + "\n"); + SyslogServerTCP myServ = new + SyslogServerTCP(Integer.parseInt(args[0]), cons); + myServ.start(); + System.out.println("Press ctl-c to terminate\n"); + } + catch(Exception e) { + System.out.println("Error: " + e.toString()); + } + } +} diff --git a/java/com/rsyslog/gui/simpServ/simpServConsumer.java b/java/com/rsyslog/gui/simpServ/simpServConsumer.java new file mode 100644 index 00000000..588f2640 --- /dev/null +++ b/java/com/rsyslog/gui/simpServ/simpServConsumer.java @@ -0,0 +1,32 @@ +/** A syslog message consumer for the simple syslog server. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.simpServ; +import com.rsyslog.lib.*; + +class simpServConsumer implements SyslogMsgConsumer { + public void consumeMsg(String ln) { + SyslogMessage msg = new SyslogMessage(ln); + System.out.println("Line received '" + msg.getRawMsgAfterPRI() + "'\n"); + } +} diff --git a/java/com/rsyslog/lib/DiagSess.java b/java/com/rsyslog/lib/DiagSess.java new file mode 100644 index 00000000..799b9a4a --- /dev/null +++ b/java/com/rsyslog/lib/DiagSess.java @@ -0,0 +1,78 @@ +/* The diagnostic session to an imdiag module (running inside rsyslogd). + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; +import java.io.*; +import java.net.*; + +public class DiagSess { + + private String host = new String("127.0.0.1"); + private int port = 13500; + int timeout = 0; + private PrintWriter out = null; + private BufferedReader in = null; + private Socket diagSocket = null; + + /** set connection timeout */ + public void setTimeout(int timeout_) { + timeout = timeout_; + } + + public DiagSess(String host_, int port_) { + host = host_; + port = port_; + } + + /** connect to remote server. Initializes everything for request-reply + * processing. + * + * @throws IOException + */ + public void connect() throws IOException { + diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(timeout); + out = new PrintWriter(diagSocket.getOutputStream(), true); + in = new BufferedReader(new InputStreamReader( + diagSocket.getInputStream())); + + } + + /** end session with remote server. */ + public void disconnect() throws IOException { + out.close(); + in.close(); + diagSocket.close(); + } + + /** issue a request to imdiag and return its response. + * + * @param req request string + * @return response string (unparsed) + * @throws IOException + */ + public String request(String req) throws IOException { + out.println(req); + String resp = in.readLine(); + return resp; + } + +} diff --git a/java/com/rsyslog/lib/SyslogMessage.java b/java/com/rsyslog/lib/SyslogMessage.java new file mode 100644 index 00000000..b544a6db --- /dev/null +++ b/java/com/rsyslog/lib/SyslogMessage.java @@ -0,0 +1,75 @@ +/** + * Implementation of the syslog message object. + * + * This is a limited-capability implementation of a syslog message together + * with all its properties. It is limit to what is currently needed and may + * be extended as further need arises. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public class SyslogMessage { + + /** message as received from the wire */ + private String rawmsg; + /** the rawmsg without the PRI part */ + private String rawMsgAfterPRI; + /** PRI part */ + private int pri; + + /** a very simple syslog parser. So far, it only parses out the + * PRI part of the message. May be extended later. Rawmsg must have + * been set before the parser is called. It will populate "all" other + * fields. + */ + private void parse() { + int i; + if(rawmsg.charAt(0) == '<') { + pri = 0; + for(i = 1 ; Character.isDigit(rawmsg.charAt(i)) && i < 4 ; ++i) { + pri = pri * 10 + rawmsg.charAt(i) - '0'; + } + if(rawmsg.charAt(i) != '>') + /* not a real cure, but sufficient for the current + * mini-parser... */ + --i; + rawMsgAfterPRI = rawmsg.substring(i + 1); + } else { + pri = 116; + rawMsgAfterPRI = rawmsg; + } + } + + public SyslogMessage(String _rawmsg) { + rawmsg = _rawmsg; + parse(); + } + + public String getRawMsg() { + return rawmsg; + } + + public String getRawMsgAfterPRI() { + return rawMsgAfterPRI; + } +} diff --git a/java/com/rsyslog/lib/SyslogMsgConsumer.java b/java/com/rsyslog/lib/SyslogMsgConsumer.java new file mode 100644 index 00000000..42c9931a --- /dev/null +++ b/java/com/rsyslog/lib/SyslogMsgConsumer.java @@ -0,0 +1,29 @@ +/** + * Interface for SyslogConsumers. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public interface SyslogMsgConsumer { + public void consumeMsg(String msg); +} diff --git a/java/com/rsyslog/lib/SyslogSender.java b/java/com/rsyslog/lib/SyslogSender.java new file mode 100644 index 00000000..fc0e3fec --- /dev/null +++ b/java/com/rsyslog/lib/SyslogSender.java @@ -0,0 +1,96 @@ +/** + * This class specifies all methods common to syslog senders. It also implements + * some generic ways to send data. Actual syslog senders (e.g. UDP, TCP, ...) shall + * be derived from this class. + * + * This is a limited-capability implementation of a syslog message together + * with all its properties. It is limit to what is currently needed and may + * be extended as further need arises. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public abstract class SyslogSender { + + /** the rawmsg without the PRI part */ + private String target; + + /** the rawmsg without the PRI part */ + private boolean isConnected = false; + + /** Constructs Sender, sets target system. + * @param target the system to connect to. Syntax of target is depending + * on the underlying transport. + */ + public SyslogSender(String target) { + this.target = target; + } + + + /** send a message on the wire. + * This needs a complete formatted message, which will be extended by + * the transport framing, if necessary. + * + * @param MSG a validly formatted syslog message as of the RFC (all parts) + * @throws Exception (depending on transport) + */ + protected abstract void sendTransport(String MSG) throws Exception; + + /** send an alread-formatted message. + * Sends a preformatted syslog message payload to the target. Connects + * to the target if not already connected. + * + * @param MSG a validly formatted syslog message as of the RFC (all parts) + * @throws Exception (depending on transport) + */ + public void sendMSG(String MSG) throws Exception { + if(!isConnected) + connect(); + sendTransport(MSG); + } + + /** connect to the target. + * Note that this may be a null operation if there is no session-like entity + * in the underlying transport (as is for example in UDP). + */ + public void connect() throws Exception { + /* the default implementation does (almost) nothing */ + isConnected = true; + } + + /** disconnects from the target. + * Note that this may be a null operation if there is no session-like entity + * in the underlying transport (as is for example in UDP). + */ + public void disconnect() { + /* the default implementation does (almost) nothing */ + isConnected = false; + } + + /** return target of this Sender. + * @returns target as initially set + */ + public String getTarget() { + return target; + } +} diff --git a/java/com/rsyslog/lib/SyslogServerTCP.java b/java/com/rsyslog/lib/SyslogServerTCP.java new file mode 100644 index 00000000..d5376a32 --- /dev/null +++ b/java/com/rsyslog/lib/SyslogServerTCP.java @@ -0,0 +1,126 @@ +/** + * Implementation of a tcp-based syslog server. + * + * This is a limited-capability implementation of a syslog tcp server. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +import com.rsyslog.lib.SyslogMsgConsumer; +import java.io.*; +import java.net.*; + + +/** a small test consumer */ +/* +class TestConsumer implements SyslogMsgConsumer { + public void consumeMsg(String ln) { + System.out.println("Line received '" + ln + "'\n"); + } +} +*/ + +public class SyslogServerTCP extends Thread { + private ServerSocket lstnSock; + private boolean contRun; /* continue processing requests? */ + public SyslogMsgConsumer consumer; + + /** Process a single connection */ + class Session extends Thread { + private Socket sock; + private SyslogServerTCP srvr; + + public Session(Socket so, SyslogServerTCP _srvr) { + sock = so; + srvr = _srvr; + } + + public void run() { + try { + BufferedReader data = new BufferedReader( + new InputStreamReader(sock.getInputStream())); + + String ln = data.readLine(); + while(ln != null) { + srvr.getConsumer().consumeMsg(ln); + ln = data.readLine(); + } + System.out.println("End of Session.\n"); + sock.close(); + } + catch(Exception e) { + /* we ignore any errors we may have... */ + System.out.println("Session exception " + e.toString()); + } + } + } + + /** a small test driver */ +/* + public static void main(String args[]) { + try { + SyslogMsgConsumer cons = new TestConsumer(); + System.out.println("Starting server on port " + args[0] + "\n"); + SyslogServerTCP myServ = new + SyslogServerTCP(Integer.parseInt(args[0]), cons); + myServ.start(); + System.out.println("Press ctl-c to terminate\n"); + } + catch(Exception e) { + System.out.println("Fehler! " + e.toString()); + } + } +*/ + + public SyslogServerTCP(int port, SyslogMsgConsumer cons) throws java.io.IOException { + if(lstnSock != null) + terminate(); + lstnSock = new ServerSocket(port); + consumer = cons; + contRun = true; + } + + public void terminate() { + contRun = false; + } + + public SyslogMsgConsumer getConsumer() { + return consumer; + } + + public void run() { + try { + while(contRun) { + Socket sock = lstnSock.accept(); + System.out.println("New connection request! " + sock.toString()); + Thread sess = new Session(sock, this); + sock = null; + sess.start(); + } + } + catch(Exception e) { + System.out.println("Error during server run " + e.toString()); + } + + } +} diff --git a/java/com/rsyslog/lib/SyslogTrafficGenerator.java b/java/com/rsyslog/lib/SyslogTrafficGenerator.java new file mode 100644 index 00000000..79a99495 --- /dev/null +++ b/java/com/rsyslog/lib/SyslogTrafficGenerator.java @@ -0,0 +1,81 @@ +/** + * This class is a syslog traffic generator. It is primarily intended to be used + * together with testing tools, but may have some use cases outside that domain. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public class SyslogTrafficGenerator extends Thread { + + /** the target host to receive traffic */ + private String target; + + /** the message (template) to be sent */ + private String message; + + /** number of messages to be sent */ + private long nummsgs; + + /** Constructs Sender, sets target system. + * @param target the system to connect to. Syntax of target is depending + * on the underlying transport. + */ + public SyslogTrafficGenerator(String target, String message, long nummsgs) { + this.target = target; + this.message = message; + this.nummsgs = nummsgs; + } + + /** Generates the traffic. Stops when either called to terminate + * or the max number of messages have been sent. Note that all + * necessary properties must have been set up before starting the + * generator thread! + */ + private void performTest() throws Exception { + int doDisp = 0; + UDPSyslogSender sender = new UDPSyslogSender(target); + for(long i = 0 ; i < nummsgs ; ++i) { + sender.sendMSG(message + " " + Long.toString(i) + " " + this.toString() + "\0"); + if((doDisp++ % 1000) == 0) + System.out.println(this.toString() + " send message " + Long.toString(i)); + sleep(1); + } + } + + +/** Wrapper around the real traffic generator, catches exceptions. + */ + public void run() { + System.out.println("traffic generator " + this.toString() + " thread started"); + try { + performTest(); + } + catch(Exception e) { + /* at some time, we may find a more intelligent way to + * handle this! ;) + */ + System.out.println(e.toString()); + } + System.out.println("traffic generator " + this.toString() + " thread finished"); + } +} diff --git a/java/com/rsyslog/lib/UDPSyslogSender.java b/java/com/rsyslog/lib/UDPSyslogSender.java new file mode 100644 index 00000000..1a2c4726 --- /dev/null +++ b/java/com/rsyslog/lib/UDPSyslogSender.java @@ -0,0 +1,75 @@ +/** + * A UDP transport implementation of a syslog sender. + * + * Note that there is an anomaly in this version of the code: we query the remote system + * address only once during the connection setup and resue it. If we potentially run for + * an extended period of time, the remote address may change, what we do not reflect. For + * the current use case, this is acceptable, but if this code is put into more wide-spread + * use outside of debugging, a periodic requery should be added. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; +import java.net.*; + +public class UDPSyslogSender extends SyslogSender { + + private final int port = 514; // TODO: take from target! + private InetAddress targetAddr; + + /** the socket to communicate over with the remote system. */ + private DatagramSocket sock; + + /** Constructs Sender, sets target system. + * @param target the system to connect to. Syntax of target is depending + * on the underlying transport. + */ + public UDPSyslogSender(String target) throws Exception { + super(target); + } + + /** send a message on the wire. + * This needs a complete formatted message, which will be extended by + * the transport framing, if necessary. + * + * @param MSG a validly formatted syslog message as of the RFC (all parts) + * @throws Exception (depending on transport) + */ + protected void sendTransport(String MSG) throws Exception { + byte msg[] = MSG.getBytes(); + DatagramPacket pkt = new DatagramPacket(msg, msg.length, targetAddr, port); + sock.send(pkt); + } + + + /** connect to the target. + * For UDP, this means we create the socket. + */ + public void connect() throws Exception { + super.connect(); + sock = new DatagramSocket(); + + // TODO: we should extract the actual hostname & port! + targetAddr = InetAddress.getByName(getTarget()); + } + +} diff --git a/m4/atomic_operations_64bit.m4 b/m4/atomic_operations_64bit.m4 new file mode 100644 index 00000000..3121cf15 --- /dev/null +++ b/m4/atomic_operations_64bit.m4 @@ -0,0 +1,53 @@ +# rsyslog +# +# atomic_operations.m4 - autoconf macro to check if compiler supports atomic +# operations +# +# rgerhards, 2008-09-18, added based on +# http://svn.apache.org/repos/asf/apr/apr/trunk/configure.in +# +# +AC_DEFUN([RS_ATOMIC_OPERATIONS_64BIT], +[AC_CACHE_CHECK([whether the compiler provides atomic builtins for 64 bit data types], [ap_cv_atomic_builtins_64], +[AC_TRY_RUN([ +int main() +{ + unsigned long long val = 1010, tmp, *mem = &val; + + if (__sync_fetch_and_add(&val, 1010) != 1010 || val != 2020) + return 1; + + tmp = val; + + if (__sync_fetch_and_sub(mem, 1010) != tmp || val != 1010) + return 1; + + if (__sync_sub_and_fetch(&val, 1010) != 0 || val != 0) + return 1; + + tmp = 3030; + + if (__sync_val_compare_and_swap(mem, 0, tmp) != 0 || val != tmp) + return 1; + + if (__sync_lock_test_and_set(&val, 4040) != 3030) + return 1; + + mem = &tmp; + + if (__sync_val_compare_and_swap(&mem, &tmp, &val) != &tmp) + return 1; + + __sync_synchronize(); + + if (mem != &val) + return 1; + + return 0; +}], [ap_cv_atomic_builtins_64=yes], [ap_cv_atomic_builtins_64=no], [ap_cv_atomic_builtins_64=no])]) + +if test "$ap_cv_atomic_builtins_64" = "yes"; then + AC_DEFINE(HAVE_ATOMIC_BUILTINS_64BIT, 1, [Define if compiler provides 64 bit atomic builtins]) +fi + +]) diff --git a/m4/shave.m4 b/m4/shave.m4 deleted file mode 100644 index e647e579..00000000 --- a/m4/shave.m4 +++ /dev/null @@ -1,99 +0,0 @@ -dnl Make automake/libtool output more friendly to humans -dnl -dnl Copyright (c) 2009, Damien Lespiau <damien.lespiau@gmail.com> -dnl -dnl Permission is hereby granted, free of charge, to any person -dnl obtaining a copy of this software and associated documentation -dnl files (the "Software"), to deal in the Software without -dnl restriction, including without limitation the rights to use, -dnl copy, modify, merge, publish, distribute, sublicense, and/or sell -dnl copies of the Software, and to permit persons to whom the -dnl Software is furnished to do so, subject to the following -dnl conditions: -dnl -dnl The above copyright notice and this permission notice shall be -dnl included in all copies or substantial portions of the Software. -dnl -dnl THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -dnl EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -dnl OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -dnl NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -dnl HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -dnl WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -dnl FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -dnl OTHER DEALINGS IN THE SOFTWARE. -dnl -dnl SHAVE_INIT([shavedir],[default_mode]) -dnl -dnl shavedir: the directory where the shave scripts are, it defaults to -dnl $(top_builddir) -dnl default_mode: (enable|disable) default shave mode. This parameter -dnl controls shave's behaviour when no option has been -dnl given to configure. It defaults to disable. -dnl -dnl * SHAVE_INIT should be called late in your configure.(ac|in) file (just -dnl before AC_CONFIG_FILE/AC_OUTPUT is perfect. This macro rewrites CC and -dnl LIBTOOL, you don't want the configure tests to have these variables -dnl re-defined. -dnl * This macro requires GNU make's -s option. - -AC_DEFUN([_SHAVE_ARG_ENABLE], -[ - AC_ARG_ENABLE([shave], - AS_HELP_STRING( - [--enable-shave], - [use shave to make the build pretty [[default=$1]]]),, - [enable_shave=$1] - ) -]) - -AC_DEFUN([SHAVE_INIT], -[ - dnl you can tweak the default value of enable_shave - m4_if([$2], [enable], [_SHAVE_ARG_ENABLE(yes)], [_SHAVE_ARG_ENABLE(no)]) - - if test x"$enable_shave" = xyes; then - dnl where can we find the shave scripts? - m4_if([$1],, - [shavedir="$ac_pwd"], - [shavedir="$ac_pwd/$1"]) - AC_SUBST(shavedir) - - dnl make is now quiet - AC_SUBST([MAKEFLAGS], [-s]) - AC_SUBST([AM_MAKEFLAGS], ['`test -z $V && echo -s`']) - - dnl we need sed - AC_CHECK_PROG(SED,sed,sed,false) - - dnl substitute libtool - SHAVE_SAVED_LIBTOOL=$LIBTOOL - LIBTOOL="${SHELL} ${shavedir}/shave-libtool '${SHAVE_SAVED_LIBTOOL}'" - AC_SUBST(LIBTOOL) - - dnl substitute cc/cxx - SHAVE_SAVED_CC=$CC - SHAVE_SAVED_CXX=$CXX - SHAVE_SAVED_FC=$FC - SHAVE_SAVED_F77=$F77 - SHAVE_SAVED_OBJC=$OBJC - CC="${SHELL} ${shavedir}/shave cc ${SHAVE_SAVED_CC}" - CXX="${SHELL} ${shavedir}/shave cxx ${SHAVE_SAVED_CXX}" - FC="${SHELL} ${shavedir}/shave fc ${SHAVE_SAVED_FC}" - F77="${SHELL} ${shavedir}/shave f77 ${SHAVE_SAVED_F77}" - OBJC="${SHELL} ${shavedir}/shave objc ${SHAVE_SAVED_OBJC}" - AC_SUBST(CC) - AC_SUBST(CXX) - AC_SUBST(FC) - AC_SUBST(F77) - AC_SUBST(OBJC) - - V=@ - else - V=1 - fi - Q='$(V:1=)' - AC_SUBST(V) - AC_SUBST(Q) -]) - diff --git a/outchannel.c b/outchannel.c index 5c348b63..b9b4a6b1 100644 --- a/outchannel.c +++ b/outchannel.c @@ -38,6 +38,7 @@ #include "stringbuf.h" #include "outchannel.h" #include "dirty.h" +#include "debug.h" static struct outchannel *ochRoot = NULL; /* the root of the outchannel list */ static struct outchannel *ochLast = NULL; /* points to the last element of the outchannel list */ @@ -105,22 +106,21 @@ static rsRetVal get_Field(uchar **pp, uchar **pField) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); - rsCStrSetAllocIncrement(pStrB, 32); + CHKiRet(cstrConstruct(&pStrB)); /* copy the field */ while(*p && *p != ' ' && *p != ',') { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pField, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pField, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -174,22 +174,21 @@ static inline rsRetVal get_restOfLine(uchar **pp, uchar **pBuf) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); - rsCStrSetAllocIncrement(pStrB, 32); + CHKiRet(cstrConstruct(&pStrB)); /* copy the field */ while(*p) { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pBuf, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pBuf, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -214,7 +213,7 @@ struct outchannel *ochAddLine(char* pName, uchar** ppRestOfConfLine) return NULL; pOch->iLenName = strlen(pName); - pOch->pszName = (char*) malloc(sizeof(char) * (pOch->iLenName + 1)); + pOch->pszName = (char*) MALLOC(sizeof(char) * (pOch->iLenName + 1)); if(pOch->pszName == NULL) { dbgprintf("ochAddLine could not alloc memory for outchannel name!"); pOch->iLenName = 0; @@ -37,6 +37,7 @@ #include "rsyslog.h" #include "net.h" /* struct NetAddr */ #include "parse.h" +#include "debug.h" /* ################################################################# * * private members * @@ -256,7 +257,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != cDelim) { - CHKiRet(rsCStrAppendChar(pCStr, bConvLower ? tolower(*pC) : *pC)); + CHKiRet(cstrAppendChar(pCStr, bConvLower ? tolower(*pC) : *pC)); ++pThis->iCurrPos; ++pC; } @@ -268,10 +269,10 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim /* We got the string, now take it and see if we need to * remove anything at its end. */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); if(bTrimTrailing) { - CHKiRet(rsCStrTrimTrailingWhiteSpace(pCStr)); + CHKiRet(cstrTrimTrailingWhiteSpace(pCStr)); } /* done! */ @@ -313,23 +314,23 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; /* OK, we most probably can obtain a value... */ - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); - while(pThis->iCurrPos < rsCStrLen(pThis->pCStr)) { + while(pThis->iCurrPos < cstrLen(pThis->pCStr)) { if(*pC == '"') { break; /* we are done! */ } else if(*pC == '\\') { ++pThis->iCurrPos; ++pC; - if(pThis->iCurrPos < rsCStrLen(pThis->pCStr)) { + if(pThis->iCurrPos < cstrLen(pThis->pCStr)) { /* in this case, we copy the escaped character * to the output buffer (but do not rely on this, * we might later introduce other things, like \007! */ - CHKiRet(rsCStrAppendChar(pCStr, *pC)); + CHKiRet(cstrAppendChar(pCStr, *pC)); } } else { /* regular character */ - CHKiRet(rsCStrAppendChar(pCStr, *pC)); + CHKiRet(cstrAppendChar(pCStr, *pC)); } ++pThis->iCurrPos; ++pC; @@ -339,12 +340,12 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) ++pThis->iCurrPos; /* 'eat' trailing quote */ } else { /* error - improperly quoted string! */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); ABORT_FINALIZE(RS_RET_MISSING_TRAIL_QUOTE); } /* We got the string, let's finish it... */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* done! */ *ppCStr = pCStr; @@ -352,7 +353,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) finalize_it: if(iRet != RS_RET_OK) { if(pCStr != NULL) - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); } RETiRet; @@ -380,7 +381,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) assert(pIP != NULL); assert(pBits != NULL); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); parsSkipWhitespace(pThis); pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; @@ -390,8 +391,8 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) */ while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != '/' && *pC != ',' && !isspace((int)*pC)) { - if((iRet = rsCStrAppendChar(pCStr, *pC)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrAppendChar(pCStr, *pC)) != RS_RET_OK) { + cstrDestruct (&pCStr); FINALIZE; } ++pThis->iCurrPos; @@ -399,15 +400,15 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) } /* We got the string, let's finish it... */ - if((iRet = rsCStrFinish(pCStr)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrFinalize(pCStr)) != RS_RET_OK) { + cstrDestruct(&pCStr); FINALIZE; } /* now we have the string and must check/convert it to * an NetAddr structure. */ - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pszIP, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pszIP, 0)); *pIP = calloc(1, sizeof(struct NetAddr)); @@ -429,7 +430,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) switch(getaddrinfo ((char*)pszIP+1, NULL, &hints, &res)) { case 0: - (*pIP)->addr.NetAddr = malloc (res->ai_addrlen); + (*pIP)->addr.NetAddr = MALLOC (res->ai_addrlen); memcpy ((*pIP)->addr.NetAddr, res->ai_addr, res->ai_addrlen); freeaddrinfo (res); break; @@ -468,7 +469,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) switch(getaddrinfo ((char*)pszIP, NULL, &hints, &res)) { case 0: - (*pIP)->addr.NetAddr = malloc (res->ai_addrlen); + (*pIP)->addr.NetAddr = MALLOC (res->ai_addrlen); memcpy ((*pIP)->addr.NetAddr, res->ai_addr, res->ai_addrlen); freeaddrinfo (res); break; diff --git a/plugins/cust1/Makefile.am b/plugins/cust1/Makefile.am new file mode 100644 index 00000000..d2e075f9 --- /dev/null +++ b/plugins/cust1/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = cust1.la + +cust1_la_SOURCES = cust1.c +cust1_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +cust1_la_LDFLAGS = -module -avoid-version +cust1_la_LIBADD = diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 40f94692..81b357ef 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -52,6 +52,8 @@ #include "errmsg.h" #include "tcpsrv.h" #include "srUtils.h" +#include "msg.h" +#include "datetime.h" #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT @@ -63,10 +65,15 @@ DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) +DEFobjCurrIf(prop) /* Module static data */ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ static permittedPeers_t *pPermPeersRoot = NULL; +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this input */ +static prop_t *pRcvDummy = NULL; +static prop_t *pRcvIPDummy = NULL; /* config settings */ @@ -134,35 +141,157 @@ onErrClose(tcps_sess_t *pSess) /* ------------------------------ end callbacks ------------------------------ */ +/* get the first word delimited by space from a given string. The pointer is + * advanced to after the word. Any leading spaces are discarded. If the + * output buffer is too small, parsing ends on buffer full condition. + * An empty buffer is returned if there is no more data inside the string. + * rgerhards, 2009-05-27 + */ +#define TO_LOWERCASE 1 +#define NO_MODIFY 0 +static void +getFirstWord(uchar **ppszSrc, uchar *pszBuf, size_t lenBuf, int options) +{ + uchar c; + uchar *pszSrc = *ppszSrc; + + while(*pszSrc && *pszSrc == ' ') + ++pszSrc; /* skip to first non-space */ + + while(*pszSrc && *pszSrc != ' ' && lenBuf > 1) { + c = *pszSrc++; + if(options & TO_LOWERCASE) + c = tolower(c); + *pszBuf++ = c; + lenBuf--; + } + + *pszBuf = '\0'; + *ppszSrc = pszSrc; +} + + +/* send a response back to the originator + * rgerhards, 2009-05-27 + */ +static rsRetVal __attribute__((format(printf, 2, 3))) +sendResponse(tcps_sess_t *pSess, char *fmt, ...) +{ + va_list ap; + ssize_t len; + uchar buf[1024]; + DEFiRet; + + va_start(ap, fmt); + len = vsnprintf((char*)buf, sizeof(buf), fmt, ap); + va_end(ap); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + +finalize_it: + RETiRet; +} + + +/* actually submit a message to the rsyslog core + */ +static rsRetVal +doInjectMsg(int iNum) +{ + uchar szMsg[1024]; + msg_t *pMsg; + struct syslogTime stTime; + time_t ttGenTime; + DEFiRet; + + snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum); + + datetime.getCurrTime(&stTime, &ttGenTime); + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + MsgSetRawMsg(pMsg, (char*) szMsg, ustrlen(szMsg)); + MsgSetInputName(pMsg, pInputName); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; + MsgSetRcvFrom(pMsg, pRcvDummy); + CHKiRet(MsgSetRcvFromIP(pMsg, pRcvIPDummy)); + CHKiRet(submitMsg(pMsg)); + +finalize_it: + RETiRet; +} + + +/* This function injects messages. Command format: + * injectmsg <fromnbr> <number-of-messages> + * rgerhards, 2009-05-27 + */ +static rsRetVal +injectMsg(uchar *pszCmd, tcps_sess_t *pSess) +{ + uchar wordBuf[1024]; + int iFrom; + int nMsgs; + int i; + DEFiRet; + + /* we do not check errors here! */ + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); + iFrom = atoi((char*)wordBuf); + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); + nMsgs = atoi((char*)wordBuf); + + for(i = 0 ; i < nMsgs ; ++i) { + doInjectMsg(i + iFrom); + } + + CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); + +finalize_it: + RETiRet; +} + + /* This function waits until the main queue is drained (size = 0) */ static rsRetVal -waitMainQEmpty(void) +waitMainQEmpty(tcps_sess_t *pSess) { int iMsgQueueSize; + int iPrint = 0; DEFiRet; CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); while(iMsgQueueSize > 0) { + /* DEV DEBUG ONLY if(iPrint++ % 500) + printf("imdiag: main msg queue size: %d\n", iMsgQueueSize); + */ + if(iPrint++ % 500 == 0) + dbgprintf("imdiag sleeping, wait mainq drain, curr size %d\n", iMsgQueueSize); srSleep(0,2); /* wait a little bit */ CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + if(iMsgQueueSize == 0) { + /* verify that queue is still empty (else it could just be a race!) */ + srSleep(1,5); /* wait a little bit */ + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + } } + CHKiRet(sendResponse(pSess, "mainqueue empty\n")); + finalize_it: RETiRet; } - /* Function to handle received messages. This is our core function! * rgerhards, 2009-05-24 */ static rsRetVal OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) { - ssize_t len; int iMsgQueueSize; uchar *pszMsg; - uchar buf[1024]; + uchar cmdBuf[1024]; DEFiRet; assert(pSess != NULL); @@ -172,21 +301,23 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) * WITHOUT a termination \0 char. So we need to convert it to one * before proceeding. */ - CHKmalloc(pszMsg = malloc(sizeof(uchar) * (iLenMsg + 1))); + CHKmalloc(pszMsg = MALLOC(sizeof(uchar) * (iLenMsg + 1))); memcpy(pszMsg, pRcv, iLenMsg); pszMsg[iLenMsg] = '\0'; - if(!ustrcmp(pszMsg, UCHAR_CONSTANT("GetMainMsgQueueSize"))) { + getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf)/sizeof(uchar), TO_LOWERCASE); + + dbgprintf("imdiag received command '%s'\n", cmdBuf); + if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "%d\n", iMsgQueueSize); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); - } else if(!ustrcmp(pszMsg, UCHAR_CONSTANT("WaitMainQueueEmpty"))) { - CHKiRet(waitMainQEmpty()); - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "mainqueue empty\n"); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); + } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) { + CHKiRet(waitMainQEmpty(pSess)); + } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) { + CHKiRet(injectMsg(pszMsg, pSess)); } else { - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "unkown command '%s'\n", pszMsg); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + dbgprintf("imdiag unkown command '%s'\n", cmdBuf); + CHKiRet(sendResponse(pSess, "unkown command '%s'\n", cmdBuf)); } finalize_it: @@ -260,13 +391,31 @@ CODESTARTwillRun /* first apply some config settings */ if(pOurTcpsrv == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imdiag"), sizeof("imdiag") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + + CHKiRet(prop.Construct(&pRcvDummy)); + CHKiRet(prop.SetString(pRcvDummy, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pRcvDummy)); + + CHKiRet(prop.Construct(&pRcvIPDummy)); + CHKiRet(prop.SetString(pRcvIPDummy, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pRcvIPDummy)); + finalize_it: ENDwillRun BEGINafterRun CODESTARTafterRun - /* do cleanup here */ + if(pInputName != NULL) + prop.Destruct(&pInputName); + if(pRcvDummy != NULL) + prop.Destruct(&pRcvDummy); + if(pRcvIPDummy != NULL) + prop.Destruct(&pRcvIPDummy); ENDafterRun @@ -285,6 +434,8 @@ CODESTARTmodExit objRelease(tcps_sess, LM_TCPSRV_FILENAME); objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDmodExit @@ -303,10 +454,17 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus } +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt @@ -321,6 +479,8 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverrun"), 0, eCmdHdlrGetWord, diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 92fd30c3..8a10e26f 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -5,7 +5,7 @@ * * Work originally begun on 2008-02-01 by Rainer Gerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -46,6 +46,7 @@ #include "glbl.h" #include "datetime.h" #include "unicode-helper.h" +#include "prop.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -56,10 +57,13 @@ DEF_IMOD_STATIC_DATA /* must be present, starts static data */ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) +DEFobjCurrIf(strm) +DEFobjCurrIf(prop) typedef struct fileInfo_s { uchar *pszFileName; uchar *pszTag; + size_t lenTag; uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; @@ -79,6 +83,7 @@ static int iFilPtr = 0; /* number of files to be monitored; pointer to next fre #define MAX_INPUT_FILES 100 static fileInfo_t files[MAX_INPUT_FILES]; +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this input */ /* enqueue the read file line as a message. The provided string is * not freed - thuis must be done by the caller. @@ -95,15 +100,13 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile")); - MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); - MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); - MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); - MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); - MsgSetTAG(pMsg, (char*)pInfo->pszTag); + MsgSetInputName(pMsg, pInputName); + MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); + MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); + MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); - pMsg->bParseHOSTNAME = 0; CHKiRet(submitMsg(pMsg)); finalize_it: RETiRet; @@ -139,16 +142,16 @@ openFile(fileInfo_t *pThis) /* If we reach this point, we have a .si file */ - CHKiRet(strmConstruct(&psSF)); - CHKiRet(strmSettOperationsMode(psSF, STREAMMODE_READ)); - CHKiRet(strmSetsType(psSF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psSF, pszSFNam, lenSFNam)); - CHKiRet(strmConstructFinalize(psSF)); + CHKiRet(strm.Construct(&psSF)); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_READ)); + CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psSF, pszSFNam, lenSFNam)); + CHKiRet(strm.ConstructFinalize(psSF)); /* read back in the object */ CHKiRet(obj.Deserialize(&pThis->pStrm, (uchar*) "strm", psSF, NULL, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->pStrm)); + CHKiRet(strm.SeekCurrOffs(pThis->pStrm)); /* OK, we could successfully read the file, so we now can request that it be deleted. * If we need it again, it will be written on the next shutdown. @@ -157,14 +160,14 @@ openFile(fileInfo_t *pThis) finalize_it: if(psSF != NULL) - strmDestruct(&psSF); + strm.Destruct(&psSF); if(iRet != RS_RET_OK) { - CHKiRet(strmConstruct(&pThis->pStrm)); - CHKiRet(strmSettOperationsMode(pThis->pStrm, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR)); - CHKiRet(strmSetFName(pThis->pStrm, pThis->pszFileName, strlen((char*) pThis->pszFileName))); - CHKiRet(strmConstructFinalize(pThis->pStrm)); + CHKiRet(strm.Construct(&pThis->pStrm)); + CHKiRet(strm.SettOperationsMode(pThis->pStrm, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR)); + CHKiRet(strm.SetFName(pThis->pStrm, pThis->pszFileName, strlen((char*) pThis->pszFileName))); + CHKiRet(strm.ConstructFinalize(pThis->pStrm)); } RETiRet; @@ -203,14 +206,14 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) /* loop below will be exited when strmReadLine() returns EOF */ while(1) { - CHKiRet(strmReadLine(pThis->pStrm, &pCStr)); + CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr)); *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ } finalize_it: - /*EMPTY - just to keep the compiler happy, do NOT remove*/; + ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ /* Note: the problem above is that pthread:cleanup_pop() is a macro which * evaluates to something like "} while(0);". So the code would become * "finalize_it: }", that is a label without a statement. The C standard does @@ -240,27 +243,12 @@ finalize_it: * IMPORTANT: the calling interface of this function can NOT be modified. It actually is * called by pthreads. The provided argument is currently not being used. */ -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ static void inputModuleCleanup(void __attribute__((unused)) *arg) { BEGINfunc -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ - - - - /* so far not needed */ - - - -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ ENDfunc } -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ /* This function is called by the framework to gather the input. The module stays @@ -288,28 +276,22 @@ BEGINrunInput int i; int bHadFileData; /* were there at least one file with data during this run? */ CODESTARTrunInput - /* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ pthread_cleanup_push(inputModuleCleanup, NULL); - while(1) { /* endless loop - do NOT break; out of it! */ - /* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ + while(1) { - do { - bHadFileData = 0; - for(i = 0 ; i < iFilPtr ; ++i) { - pollFile(&files[i], &bHadFileData); - } - } while(iFilPtr > 1 && bHadFileData == 1); /* waring: do...while()! */ + do { + bHadFileData = 0; + for(i = 0 ; i < iFilPtr ; ++i) { + pollFile(&files[i], &bHadFileData); + } + } while(iFilPtr > 1 && bHadFileData == 1); /* warning: do...while()! */ - /* Note: the additional 10ns wait is vitally important. It guards rsyslog against totally - * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any - * other valid scenario. So do not remove. -- rgerhards, 2008-02-14 - */ - srSleep(iPollInterval, 10); + /* Note: the additional 10ns wait is vitally important. It guards rsyslog against totally + * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any + * other valid scenario. So do not remove. -- rgerhards, 2008-02-14 + */ + srSleep(iPollInterval, 10); - /* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ } /*NOTREACHED*/ @@ -335,6 +317,11 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imfile"), sizeof("imfile") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + finalize_it: ENDwillRun @@ -354,21 +341,20 @@ persistStrmState(fileInfo_t *pInfo) ASSERT(pInfo != NULL); /* TODO: create a function persistObj in obj.c? */ - CHKiRet(strmConstruct(&psSF)); - CHKiRet(strmSetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSettOperationsMode(psSF, STREAMMODE_WRITE)); - CHKiRet(strmSetiAddtlOpenFlags(psSF, O_TRUNC)); - CHKiRet(strmSetsType(psSF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); - CHKiRet(strmConstructFinalize(psSF)); + CHKiRet(strm.Construct(&psSF)); + CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE_TRUNC)); + CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); + CHKiRet(strm.ConstructFinalize(psSF)); - CHKiRet(strmSerialize(pInfo->pStrm, psSF)); + CHKiRet(strm.Serialize(pInfo->pStrm, psSF)); - CHKiRet(strmDestruct(&psSF)); + CHKiRet(strm.Destruct(&psSF)); finalize_it: if(psSF != NULL) - strmDestruct(&psSF); + strm.Destruct(&psSF); RETiRet; } @@ -388,9 +374,12 @@ CODESTARTafterRun for(i = 0 ; i < iFilPtr ; ++i) { if(files[i].pStrm != NULL) { /* stream open? */ persistStrmState(&files[i]); - strmDestruct(&(files[i].pStrm)); + strm.Destruct(&(files[i].pStrm)); } } + + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -401,9 +390,11 @@ ENDafterRun BEGINmodExit CODESTARTmodExit /* release objects we used */ + objRelease(strm, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDmodExit @@ -471,6 +462,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } else { pThis->pszTag = (uchar*) strdup((char*) pszFileTag); + pThis->lenTag = ustrlen(pThis->pszTag); } if(pszStateFile == NULL) { @@ -512,6 +504,8 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilename", 0, eCmdHdlrGetWord, NULL, &pszFileName, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index d8791880..dd3d67e3 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -48,6 +48,7 @@ #include "dirty.h" #include "cfsysline.h" #include "module-template.h" +#include "unicode-helper.h" #include "net.h" #include "srUtils.h" #include "gss-misc.h" @@ -56,6 +57,8 @@ #include "errmsg.h" #include "netstrm.h" #include "glbl.h" +#include "debug.h" +#include "unlimited_select.h" MODULE_TYPE_INPUT @@ -176,10 +179,10 @@ isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void *pUsrSrv, void*p pGSess = (gss_sess_t*) pUsrSess; if((pGSrv->allowedMethods & ALLOWEDMETHOD_TCP) && - net.isAllowedSender((uchar*)"TCP", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"TCP", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_TCP; if((pGSrv->allowedMethods & ALLOWEDMETHOD_GSS) && - net.isAllowedSender((uchar*)"GSS", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"GSS", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_GSS; if(allowedMethods && pGSess != NULL) pGSess->allowedMethods = allowedMethods; @@ -330,6 +333,7 @@ addGSSListener(void __attribute__((unused)) *pVal, uchar *pNewVal) CHKiRet(tcpsrv.SetCBOnSessAccept(pOurTcpsrv, onSessAccept)); CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, UCHAR_CONSTANT("imgssapi"))); tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); } @@ -407,22 +411,27 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) */ char *buf; int ret = 0; - CHKmalloc(buf = (char*) malloc(sizeof(char) * (glbl.GetMaxLine() + 1))); + CHKmalloc(buf = (char*) MALLOC(sizeof(char) * (glbl.GetMaxLine() + 1))); dbgprintf("GSS-API Trying to accept TCP session %p\n", pSess); CHKiRet(netstrm.GetSock(pSess->pStrm, &fdSess)); // TODO: method access! if (allowedMethods & ALLOWEDMETHOD_TCP) { int len; - fd_set fds; struct timeval tv; +#ifdef USE_UNLIMITED_SELECT + fd_set *pFds = malloc(glbl.GetFdSetSize()); +#else + fd_set fds; + fd_set *pFds = &fds; +#endif do { - FD_ZERO(&fds); - FD_SET(fdSess, &fds); + FD_ZERO(pFds); + FD_SET(fdSess, pFds); tv.tv_sec = 1; tv.tv_usec = 0; - ret = select(fdSess + 1, &fds, NULL, NULL, &tv); + ret = select(fdSess + 1, pFds, NULL, NULL, &tv); } while (ret < 0 && errno == EINTR); if (ret < 0) { errmsg.LogError(0, RS_RET_ERR, "TCP session %p will be closed, error ignored\n", pSess); @@ -475,6 +484,8 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) pGSess->allowedMethods = ALLOWEDMETHOD_TCP; ABORT_FINALIZE(RS_RET_OK); // TODO: define good error codes } + + freeFdSet(pFds); } context = &pGSess->gss_context; @@ -674,9 +685,17 @@ CODESTARTafterRun ENDafterRun +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index 090c4e9b..b7899353 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -75,6 +75,7 @@ #include "rsyslog.h" #include "imklog.h" +#include "debug.h" /* globals */ static int fklog = -1; /* /dev/klog */ @@ -83,6 +84,11 @@ static int fklog = -1; /* /dev/klog */ # define _PATH_KLOG "/dev/klog" #endif +static uchar *GetPath(void) +{ + return pszPath ? pszPath : _PATH_KLOG; +} + /* open the kernel log - will be called inside the willRun() imklog * entry point. -- rgerhards, 2008-04-09 */ @@ -91,9 +97,9 @@ klogWillRun(void) { DEFiRet; - fklog = open(_PATH_KLOG, O_RDONLY, 0); + fklog = open(GetPath(), O_RDONLY, 0); if (fklog < 0) { - dbgprintf("can't open %s (%d)\n", _PATH_KLOG, errno); + dbgprintf("can't open %s (%d)\n", GetPath(), errno); iRet = RS_RET_ERR; // TODO: better error code } @@ -127,7 +133,7 @@ readklog(void) if((size_t) iMaxLine < sizeof(bufRcv) - 1) { pRcv = bufRcv; } else { - if((pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))) == NULL) + if((pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))) == NULL) iMaxLine = sizeof(bufRcv) - 1; /* better this than noting */ } diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index ecb6c100..c59ce04f 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -18,7 +18,7 @@ * Please note that this file replaces the klogd daemon that was * also present in pre-v3 versions of rsyslog. * - * Copyright (C) 2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2008, 2009 by Rainer Gerhards and Adiscon GmbH * * This file is part of rsyslog. * @@ -44,6 +44,7 @@ #include <string.h> #include <stdarg.h> #include <ctype.h> +#include <stdlib.h> #include "dirty.h" #include "cfsysline.h" @@ -53,6 +54,7 @@ #include "datetime.h" #include "imklog.h" #include "glbl.h" +#include "prop.h" #include "unicode-helper.h" MODULE_TYPE_INPUT @@ -61,6 +63,7 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) /* configuration settings */ int dbgPrintSymbols = 0; /* this one is extern so the helpers can access it! */ @@ -69,6 +72,8 @@ int use_syscall = 0; int symbol_lookup = 0; /* on recent kernels > 2.6, the kernel does this */ int bPermitNonKernel = 0; /* permit logging of messages not having LOG_KERN facility */ int iFacilIntMsg; /* the facility to use for internal messages (set by driver) */ +uchar *pszPath = NULL; +int console_log_level = -1; /* TODO: configuration for the following directives must be implemented. It * was not done yet because we either do not yet have a config handler for * that type or I thought it was acceptable to push it to a later stage when @@ -76,9 +81,11 @@ int iFacilIntMsg; /* the facility to use for internal messages (set by driver) * * changes resulting from that). -- rgerhards, 2007-12-20 */ char *symfile = NULL; -int console_log_level = -1; +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this module */ +static prop_t *pLocalHostIP = NULL; /* a pseudo-constant propterty for 127.0.0.1 */ + /* enqueue the the kernel message into the message queue. * The provided msg string is not freed - thus must be done * by the caller. @@ -95,18 +102,15 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog")); - MsgSetRawMsg(pMsg, (char*)msg); - MsgSetUxTradMsg(pMsg, (char*)msg); - MsgSetRawMsg(pMsg, (char*)msg); - MsgSetMSG(pMsg, (char*)msg); - MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); - MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); - MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); - MsgSetTAG(pMsg, (char*)pszTag); + MsgSetInputName(pMsg, pInputName); + MsgSetRawMsgWOSize(pMsg, (char*)msg); + MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); + MsgSetRcvFromIP(pMsg, pLocalHostIP); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); + MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); pMsg->iFacility = LOG_FAC(iFacility); pMsg->iSeverity = LOG_PRI(iSeverity); - pMsg->bParseHOSTNAME = 0; CHKiRet(submitMsg(pMsg)); finalize_it: @@ -230,13 +234,23 @@ ENDrunInput BEGINwillRun CODESTARTwillRun + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.CreateStringProp(&pInputName, UCHAR_CONSTANT("imklog"), sizeof("imklog") - 1)); + CHKiRet(prop.CreateStringProp(&pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + iRet = klogWillRun(); +finalize_it: ENDwillRun BEGINafterRun CODESTARTafterRun iRet = klogAfterRun(); + + if(pInputName != NULL) + prop.Destruct(&pInputName); + if(pLocalHostIP != NULL) + prop.Destruct(&pLocalHostIP); ENDafterRun @@ -245,6 +259,9 @@ CODESTARTmodExit /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); + if(pszPath != NULL) + free(pszPath); ENDmodExit @@ -261,6 +278,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a symfile = NULL; symbol_lookup = 0; bPermitNonKernel = 0; + if(pszPath != NULL) { + free(pszPath); + pszPath = NULL; + } iFacilIntMsg = klogFacilIntMsg(); return RS_RET_OK; } @@ -271,14 +292,17 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); iFacilIntMsg = klogFacilIntMsg(); CHKiRet(omsdRegCFSLineHdlr((uchar *)"debugprintkernelsymbols", 0, eCmdHdlrBinary, NULL, &dbgPrintSymbols, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogpath", 0, eCmdHdlrGetWord, NULL, &pszPath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogsymbollookup", 0, eCmdHdlrBinary, NULL, &symbol_lookup, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogsymbolstwice", 0, eCmdHdlrBinary, NULL, &symbols_twice, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogusesyscallinterface", 0, eCmdHdlrBinary, NULL, &use_syscall, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogpermitnonkernelfacility", 0, eCmdHdlrBinary, NULL, &bPermitNonKernel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogconsoleloglevel", 0, eCmdHdlrInt, NULL, &console_log_level, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"kloginternalmsgfacility", 0, eCmdHdlrFacility, NULL, &iFacilIntMsg, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/imklog/imklog.h b/plugins/imklog/imklog.h index 37bd58b0..c183026d 100644 --- a/plugins/imklog/imklog.h +++ b/plugins/imklog/imklog.h @@ -52,6 +52,7 @@ extern int symbol_lookup; extern char *symfile; extern int console_log_level; extern int dbgPrintSymbols; +extern uchar *pszPath; /* the functions below may be called by the drivers */ rsRetVal imklogLogIntMsg(int priority, char *fmt, ...) __attribute__((format(printf,2, 3))); diff --git a/plugins/imklog/ksym.c b/plugins/imklog/ksym.c index f636a7bb..058b2cfa 100644 --- a/plugins/imklog/ksym.c +++ b/plugins/imklog/ksym.c @@ -122,6 +122,7 @@ #include "imklog.h" #include "ksyms.h" #include "module.h" +#include "debug.h" int num_syms = 0; @@ -523,7 +524,7 @@ static int AddSymbol(unsigned long address, char *symbol) return(0); /* Then the space for the symbol. */ - sym_array[num_syms].name = (char *) malloc(strlen(symbol)*sizeof(char) + 1); + sym_array[num_syms].name = (char *) MALLOC(strlen(symbol)*sizeof(char) + 1); if ( sym_array[num_syms].name == NULL ) return(0); diff --git a/plugins/imklog/ksym_mod.c b/plugins/imklog/ksym_mod.c index be5fdee9..82978892 100644 --- a/plugins/imklog/ksym_mod.c +++ b/plugins/imklog/ksym_mod.c @@ -106,6 +106,7 @@ #include "rsyslog.h" #include "imklog.h" #include "ksyms.h" +#include "debug.h" #define KSYMS "/proc/kallsyms" @@ -289,7 +290,7 @@ struct Module *AddModule(module) struct Module *mp; if ( num_modules == 0 ) { - sym_array_modules = (struct Module *)malloc(sizeof(struct Module)); + sym_array_modules = (struct Module *)MALLOC(sizeof(struct Module)); if ( sym_array_modules == NULL ) { diff --git a/plugins/imklog/linux.c b/plugins/imklog/linux.c index 0dd4320d..727708a5 100644 --- a/plugins/imklog/linux.c +++ b/plugins/imklog/linux.c @@ -37,6 +37,7 @@ #include "msg.h" #include "module-template.h" #include "imklog.h" +#include "unicode-helper.h" /* Includes. */ @@ -84,19 +85,21 @@ static enum LOGSRC {none, proc, kernel} logsrc; extern int ksyslog(int type, char *buf, int len); +static uchar *GetPath(void) +{ + return pszPath ? pszPath : UCHAR_CONSTANT(_PATH_KLOG); +} + static void CloseLogSrc(void) { - /* Turn on logging of messages to console, but only if we had the -c - * option -- rgerhards, 2007-08-01 - */ - if (console_log_level != -1) + /* Turn on logging of messages to console, but only if a log level was speficied */ + if(console_log_level != -1) ksyslog(7, NULL, 0); /* Shutdown the log sources. */ - switch ( logsrc ) - { + switch(logsrc) { case kernel: - ksyslog(0, 0, 0); + ksyslog(0, NULL, 0); imklogLogIntMsg(LOG_INFO, "Kernel logging (ksyslog) stopped."); break; case proc: @@ -135,7 +138,7 @@ static enum LOGSRC GetKernelLogSrc(void) * file system is available to get kernel messages from. */ if ( use_syscall || - ((stat(_PATH_KLOG, &sb) < 0) && (errno == ENOENT)) ) + ((stat((char*)GetPath(), &sb) < 0) && (errno == ENOENT)) ) { /* Initialize kernel logging. */ ksyslog(1, NULL, 0); @@ -144,14 +147,14 @@ static enum LOGSRC GetKernelLogSrc(void) return(kernel); } - if ( (kmsg = open(_PATH_KLOG, O_RDONLY|O_CLOEXEC)) < 0 ) + if ( (kmsg = open((char*)GetPath(), O_RDONLY|O_CLOEXEC)) < 0 ) { imklogLogIntMsg(LOG_ERR, "imklog: Cannot open proc file system, %d.\n", errno); - ksyslog(7, NULL, 0); /* TODO: check this, implement more */ + ksyslog(7, NULL, 0); return(none); } - imklogLogIntMsg(LOG_INFO, "imklog %s, log source = %s started.", VERSION, _PATH_KLOG); + imklogLogIntMsg(LOG_INFO, "imklog %s, log source = %s started.", VERSION, GetPath()); return(proc); } diff --git a/plugins/immark/immark.c b/plugins/immark/immark.c index 8504f872..5d48369e 100644 --- a/plugins/immark/immark.c +++ b/plugins/immark/immark.c @@ -42,6 +42,8 @@ #include "module-template.h" #include "errmsg.h" #include "msg.h" +#include "srUtils.h" +#include "glbl.h" MODULE_TYPE_INPUT @@ -50,8 +52,16 @@ MODULE_TYPE_INPUT /* Module static data */ DEF_IMOD_STATIC_DATA +DEFobjCurrIf(glbl) static int iMarkMessagePeriod = DEFAULT_MARK_PERIOD; +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + /* This function is called to gather input. It must terminate only * a) on failure (iRet set accordingly) * b) on termination of the input module (as part of the unload process) @@ -71,16 +81,13 @@ CODESTARTrunInput * right into the sleep below. */ while(1) { - /* we do not need to handle the RS_RET_TERMINATE_NOW case any - * special because we just need to terminate. This may be different - * if a cleanup is needed. But for now, we can just use CHKiRet(). - * rgerhards, 2007-12-17 - */ - CHKiRet(thrdSleep(pThrd, iMarkMessagePeriod, 0)); /* seconds, micro seconds */ + srSleep(iMarkMessagePeriod, 0); /* seconds, micro seconds */ + + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + logmsgInternal(NO_ERRCODE, LOG_INFO, (uchar*)"-- MARK --", MARK); } -finalize_it: - return iRet; ENDrunInput @@ -106,6 +113,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) @@ -119,9 +127,9 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"markmessageperiod", 0, eCmdHdlrInt, NULL, &iMarkMessagePeriod, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 2255e643..9be38f8f 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -4,7 +4,7 @@ * * File begun on 2008-03-13 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -43,15 +43,19 @@ #include "module-template.h" #include "net.h" #include "msg.h" +#include "unicode-helper.h" +#include "prop.h" MODULE_TYPE_INPUT /* static data */ DEF_IMOD_STATIC_DATA DEFobjCurrIf(net) +DEFobjCurrIf(prop) /* Module static data */ static relpEngine_t *pRelpEngine; /* our relp engine */ +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this module */ /* config settings */ @@ -77,15 +81,15 @@ isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void __attribute__((u * are different from our rsRetVal. So we can simply use our own iRet system * to fulfill the requirement. * rgerhards, 2008-03-21 - * TODO: we currently do not receive the remote hosts's IP. As a work-around, we - * use "???" for the time being. -- rgerhards, 2008-05-16 + * Note: librelp 1.0.0 is required in order to receive the IP address, otherwise + * we will only see the hostname (twice). -- rgerhards, 2009-10-14 */ static relpRetVal -onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, size_t lenMsg) +onSyslogRcv(uchar *pHostname, uchar *pIP, uchar *pMsg, size_t lenMsg) { DEFiRet; - parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, PARSE_HOSTNAME, - eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); + parseAndSubmitMessage(pHostname, pIP, pMsg, lenMsg, PARSE_HOSTNAME, + eFLOWCTL_LIGHT_DELAY, pInputName, NULL, 0); RETiRet; } @@ -130,6 +134,11 @@ CODESTARTwillRun //net.PrintAllowedSenders(2); /* TCP */ if(pRelpEngine == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); + + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imrelp"), sizeof("imrelp") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); finalize_it: ENDwillRun @@ -143,6 +152,9 @@ CODESTARTafterRun net.pAllowedSenders_TCP = NULL; } #endif + + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -152,6 +164,7 @@ CODESTARTmodExit iRet = relpEngineDestruct(&pRelpEngine); /* release objects we used */ + objRelease(prop, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); ENDmodExit @@ -177,6 +190,7 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr pRelpEngine = NULL; /* request objects we use */ + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index f0efe1ee..0cfae057 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -61,6 +61,7 @@ #include "netstrm.h" #include "errmsg.h" #include "tcpsrv.h" +#include "ruleset.h" #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT @@ -72,6 +73,7 @@ DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) DEFobjCurrIf(errmsg) +DEFobjCurrIf(ruleset) /* Module static data */ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ @@ -80,10 +82,14 @@ static permittedPeers_t *pPermPeersRoot = NULL; /* config settings */ static int iTCPSessMax = 200; /* max number of sessions */ +static int iTCPLstnMax = 20; /* max number of sessions */ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ +static int bEmitMsgOnClose = 0; /* emit an informational message on close by remote peer */ static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ +static int bDisableLFDelim = 0; /* disbale standard LF delimiter */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ +static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ /* callbacks */ @@ -92,7 +98,7 @@ static int isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) { - return net.isAllowedSender(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN); + return net.isAllowedSender2(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN, 1); } @@ -157,6 +163,27 @@ finalize_it: } +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, RS_RET_RULESET_NOT_FOUND, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imtcp current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; @@ -164,6 +191,7 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa if(pOurTcpsrv == NULL) { CHKiRet(tcpsrv.Construct(&pOurTcpsrv)); CHKiRet(tcpsrv.SetSessMax(pOurTcpsrv, iTCPSessMax)); + CHKiRet(tcpsrv.SetLstnMax(pOurTcpsrv, iTCPLstnMax)); CHKiRet(tcpsrv.SetCBIsPermittedHost(pOurTcpsrv, isPermittedHost)); CHKiRet(tcpsrv.SetCBRcvData(pOurTcpsrv, doRcvData)); CHKiRet(tcpsrv.SetCBOpenLstnSocks(pOurTcpsrv, doOpenLstnSocks)); @@ -171,6 +199,8 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); + CHKiRet(tcpsrv.SetbDisableLFDelim(pOurTcpsrv, bDisableLFDelim)); + CHKiRet(tcpsrv.SetNotificationOnRemoteClose(pOurTcpsrv, bEmitMsgOnClose)); /* now set optional params, but only if they were actually configured */ if(pszStrmDrvrAuthMode != NULL) { CHKiRet(tcpsrv.SetDrvrAuthMode(pOurTcpsrv, pszStrmDrvrAuthMode)); @@ -180,8 +210,8 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa } } -dbgprintf("XXX: try add listen port %s\n", pNewVal); - /* initialized, now add socket */ + /* initialized, now add socket and listener params */ + CHKiRet(tcpsrv.SetRuleset(pOurTcpsrv, pBindRuleset)); CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? UCHAR_CONSTANT("imtcp") : pszInputName)); tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); @@ -226,6 +256,13 @@ CODESTARTafterRun ENDafterRun +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINmodExit CODESTARTmodExit if(pOurTcpsrv != NULL) @@ -241,6 +278,7 @@ CODESTARTmodExit objRelease(tcps_sess, LM_TCPSRV_FILENAME); objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDmodExit @@ -248,16 +286,15 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { iTCPSessMax = 200; + iTCPLstnMax = 20; iStrmDrvrMode = 0; + bEmitMsgOnClose = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; - if(pszInputName != NULL) { - free(pszInputName); - pszInputName = NULL; - } - if(pszStrmDrvrAuthMode != NULL) { - free(pszStrmDrvrAuthMode); - pszStrmDrvrAuthMode = NULL; - } + bDisableLFDelim = 0; + free(pszInputName); + pszInputName = NULL; + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; return RS_RET_OK; } @@ -266,6 +303,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt @@ -280,12 +318,17 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverrun"), 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpmaxsessions"), 0, eCmdHdlrInt, NULL, &iTCPSessMax, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpmaxlisteners"), 0, eCmdHdlrInt, + NULL, &iTCPLstnMax, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpservernotifyonconnectionclose"), 0, + eCmdHdlrBinary, NULL, &bEmitMsgOnClose, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdrivermode"), 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdriverauthmode"), 0, @@ -294,8 +337,12 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserveraddtlframedelimiter"), 0, eCmdHdlrInt, NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverdisablelfdelimiter"), 0, eCmdHdlrBinary, + NULL, &bDisableLFDelim, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverinputname"), 0, eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverbindruleset"), 0, + eCmdHdlrGetWord, setRuleset, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/imtemplate/imtemplate.c b/plugins/imtemplate/imtemplate.c index 366408a0..e5e43025 100644 --- a/plugins/imtemplate/imtemplate.c +++ b/plugins/imtemplate/imtemplate.c @@ -77,6 +77,7 @@ #include "cfsysline.h" /* access to config file objects */ #include "module-template.h" /* generic module interface code - very important, read it! */ #include "srUtils.h" /* some utility functions */ +#include "debug.h" /* some debug helper functions */ MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -137,7 +138,7 @@ imtemplateMyFunc(int iMyParam) * ABORT_FINALIZE(retcode) * just like FINALIZE, except that iRet is set to the provided error * code before control is transferred, e.g. - * if((ptr = malloc(20)) == NULL) + * if((ptr = MALLOC(20)) == NULL) * ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); * * In order for all this to work, you need to define finalize_it, e.g. @@ -231,49 +232,25 @@ CODESTARTrunInput * logs an error message as syslogd, just as printf, e.g. * errmsg.LogError(NO_ERRCODE, "Error %d occured during %s", 1, "test"); * - * There are several ways how a message can be enqueued. This part of the - * interface is currently underspecified. Have a look at the function definitions - * in syslogd.c (sorry, folks...). - * - * If you received a full syslog message that must be decoded by a message - * parser, parseAndSubmitMessage() is the way to go. It's not just a funny name - * but also a quite some legacy. Consequently, its interface is, ummm, not - * well designed. - * parseAndSubmitMessage((char*)fromHost, (char*) pRcvBuf, lenRcvd, bParseHost); - * fromHost - * is the host that we received the message from (a string) - * pRcvBuf - * is the received (to-be-decoded) message - * lenRcvd - * is the length of the received message. Please note that pRcvBuf is - * NOT a standard C-string. Most importantly it is NOT expected to be - * \0-terminated. Thus the lenght is vitally imporant (if it is wrong, - * rsyslogd will probably segfault). - * bParseHost - * is a boolean (0-no, 1-yes). It tells the parser whether or not - * a hostname should be parsed from the message. This is important - * for sources that are known not to provide a hostname. - * Use define MSG_PARSE_HOSTNAME and MSG_DONT_PARSE_HOSTNAME - * - * Another, more elaborate, way is to create the message object ourselves and - * pass it to the rule engine. That way is more appropriate if the message + * To submit the message to the queue engine, we must create the message + * object and fill it with data. If it contains a syslog message that must + * be parsed, we can add a flag that requests parsing. Otherwise, we must + * fill the properties ourselves. That is appropriate if the message * does not need to be parsed, for example when reading text (log) files. In that way, * we can set the message properties as of our liking. This is how it works: * msg_t *pMsg; CHKiRet(msgConstruct(&pMsg)); - MsgSetUxTradMsg(pMsg, msg); MsgSetRawMsg(pMsg, msg); MsgSetHOSTNAME(pMsg, LocalHostName); MsgSetTAG(pMsg, "rsyslogd:"); pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - pMsg->bParseHOSTNAME = 0; flags |= INTERNAL_MSG; logmsg(pMsg, flags); / * some time, CHKiRet() will work here, too [today NOT!] * / * - * Note that UxTradMsg is a wild construct. For the time being, set it to - * the raw message text. I am hard thinking at dropping that beast at all... + * NOTE: for up-to-date usage samples, see the other provided input modules. + * A good starting point is probably imuxsock. * * This example probably does not set all message properties (but the ones * that are of practical importance). If you need all, check msg.h. Use @@ -314,7 +291,7 @@ CODESTARTwillRun if(udpLstnSocks == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); - if((pRcvBuf = malloc(glbl.GetMaxLine * sizeof(char))) == NULL) { + if((pRcvBuf = MALLOC(glbl.GetMaxLine * sizeof(char))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } * diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index f3448095..5c8e2859 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -32,6 +32,9 @@ #include <errno.h> #include <unistd.h> #include <netdb.h> +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif #include "rsyslog.h" #include "dirty.h" #include "net.h" @@ -43,6 +46,8 @@ #include "msg.h" #include "parser.h" #include "datetime.h" +#include "prop.h" +#include "ruleset.h" #include "unicode-helper.h" MODULE_TYPE_INPUT @@ -55,7 +60,10 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) DEFobjCurrIf(datetime) +DEFobjCurrIf(prop) +DEFobjCurrIf(ruleset) +static int bDoACLCheck; /* are ACL checks neeed? Cached once immediately before listener startup */ static int iMaxLine; /* maximum UDP message size supported */ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitted sender was last discarded * This shall prevent remote DoS when the "discard on disallowed sender" @@ -63,11 +71,14 @@ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitte */ static int *udpLstnSocks = NULL; /* Internet datagram sockets, first element is nbr of elements * read-only after init(), but beware of restart! */ +static ruleset_t **udpRulesets = NULL; /* ruleset to be used with sockets in question (entry 0 is empty) */ static uchar *pszBindAddr = NULL; /* IP to bind socket to */ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a global and alloc * it so that we can check available memory in willRun() and request * termination if we can not get it. -- rgerhards, 2007-12-27 */ +static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ +static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -86,6 +97,7 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) int *newSocks; int *tmpSocks; int iSrc, iDst; + ruleset_t **tmpRulesets; /* check which address to bind to. We could do this more compact, but have not * done so in order to make the code more readable. -- rgerhards, 2007-12-27 @@ -97,7 +109,7 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) else bindAddr = pszBindAddr; - dbgprintf("Trying to open syslog UDP ports at %s:%s.\n", + DBGPRINTF("Trying to open syslog UDP ports at %s:%s.\n", (bindAddr == NULL) ? (uchar*)"*" : bindAddr, pNewVal); newSocks = net.create_udp_socket(bindAddr, (pNewVal == NULL || *pNewVal == '\0') ? (uchar*) "514" : pNewVal, 1); @@ -106,26 +118,39 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) if(udpLstnSocks == NULL) { /* esay, we can just replace it */ udpLstnSocks = newSocks; + CHKmalloc(udpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (newSocks[0] + 1))); + for(iDst = 1 ; iDst <= newSocks[0] ; ++iDst) + udpRulesets[iDst] = pBindRuleset; } else { /* we need to add them */ - if((tmpSocks = malloc(sizeof(int) * 1 + newSocks[0] + udpLstnSocks[0])) == NULL) { - dbgprintf("out of memory trying to allocate udp listen socket array\n"); + tmpSocks = (int*) MALLOC(sizeof(int) * (1 + newSocks[0] + udpLstnSocks[0])); + tmpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (1 + newSocks[0] + udpLstnSocks[0])); + if(tmpSocks == NULL || tmpRulesets == NULL) { + DBGPRINTF("out of memory trying to allocate udp listen socket array\n"); /* in this case, we discard the new sockets but continue with what we * already have */ free(newSocks); + free(tmpSocks); + free(tmpRulesets); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } else { /* ready to copy */ iDst = 1; - for(iSrc = 1 ; iSrc <= udpLstnSocks[0] ; ++iSrc) - tmpSocks[iDst++] = udpLstnSocks[iSrc]; - for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc) - tmpSocks[iDst++] = newSocks[iSrc]; + for(iSrc = 1 ; iSrc <= udpLstnSocks[0] ; ++iSrc, ++iDst) { + tmpSocks[iDst] = udpLstnSocks[iSrc]; + tmpRulesets[iDst] = udpRulesets[iSrc]; + } + for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc, ++iDst) { + tmpSocks[iDst] = newSocks[iSrc]; + tmpRulesets[iDst] = pBindRuleset; + } tmpSocks[0] = udpLstnSocks[0] + newSocks[0]; free(newSocks); free(udpLstnSocks); udpLstnSocks = tmpSocks; + free(udpRulesets); + udpRulesets = tmpRulesets; } } } @@ -137,6 +162,28 @@ finalize_it: } +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal +setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imudp current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + /* This function is a helper to runInput. I have extracted it * from the main loop just so that we do not have that large amount of code * in a single place. This function takes a socket and pulls messages from @@ -152,8 +199,8 @@ finalize_it: * on scheduling order. -- rgerhards, 2008-10-02 */ static inline rsRetVal -processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, - uchar *fromHost, uchar *fromHostFQDN, uchar *fromHostIP) +processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, + ruleset_t *pRuleset) { DEFiRet; int iNbrTimeUsed; @@ -163,10 +210,15 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, ssize_t lenRcvBuf; struct sockaddr_storage frominet; msg_t *pMsg; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; char errStr[1024]; + assert(pThrd != NULL); iNbrTimeUsed = 0; while(1) { /* loop is terminated if we have a bad receive, done below in the body */ + if(pThrd->bShallStop == TRUE) + ABORT_FINALIZE(RS_RET_FORCE_TERM); socklen = sizeof(struct sockaddr_storage); lenRcvBuf = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if(lenRcvBuf < 0) { @@ -175,100 +227,167 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr); errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet"); } - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_ERR); // this most often is NOT an error, state is not checked by caller! } + if(lenRcvBuf == 0) + continue; /* this looks a bit strange, but practice shows it happens... */ + /* if we reach this point, we had a good receive and can process the packet received */ /* check if we have a different sender than before, if so, we need to query some new values */ - if(memcmp(&frominet, frominetPrev, socklen) != 0) { - CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); - memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ - /* Here we check if a host is permitted to send us - * syslog messages. If it isn't, we do not further - * process the message but log a warning (if we are - * configured to do this). - * rgerhards, 2005-09-26 - */ - *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", - (struct sockaddr *)&frominet, (char*)fromHostFQDN); - - if(!*pbIsPermitted) { - DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); - if(glbl.GetOption_DisallowWarning) { - time_t tt; - - time(&tt); - if(tt > ttLastDiscard + 60) { - ttLastDiscard = tt; - errmsg.LogError(0, NO_ERRCODE, - "UDP message from disallowed sender %s discarded", - (char*)fromHost); + if(bDoACLCheck) { + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us syslog messages. If it isn't, + * we do not further process the message but log a warning (if we are + * configured to do this). However, if the check would require name resolution, + * it is postponed to the main queue. See also my blog post at + * http://blog.gerhards.net/2009/11/acls-imudp-and-accepting-messages.html + * rgerhards, 2009-11-16 + */ + *pbIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)&frominet, "", 0); + + if(*pbIsPermitted == 0) { + DBGPRINTF("msg is not from an allowed sender\n"); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender discarded"); + } } } } + } else { + *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d)/%s,acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, fromHost, *pbIsPermitted, pRcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); - if(*pbIsPermitted) { + if(*pbIsPermitted != 0) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); } /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); - /* first trim the buffer to what we have actually received */ - CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); - memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); - pMsg->iLenRawMsg = lenRcvBuf; - MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp")); + MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); + MsgSetInputName(pMsg, pInputName); + MsgSetRuleset(pMsg, pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - pMsg->bParseHOSTNAME = 1; - MsgSetRcvFrom(pMsg, fromHost); - CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; + if(*pbIsPermitted == 2) + pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */ + CHKiRet(msgSetFromSockinfo(pMsg, &frominet)); CHKiRet(submitMsg(pMsg)); } } - finalize_it: + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; } -/* This function is called to gather input. - * Note that udpLstnSocks must be non-NULL because otherwise we would not have - * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 - * rgerhards, 2008-10-07: I have implemented a very simple, yet in most cases probably - * highly efficient "name caching". Before querying a name, I now check if the name to be - * queried is the same as the one queried in the last message processed. If that is the - * case, we can simple re-use the previous value. This algorithm works quite well with - * few sender, especially if they emit messages in bursts. The more sender and the - * more intermixed messages arrive, the less this algorithm works, but the overhead - * is so minimal (a simple memory compare and move) that this does not hurt. Even - * with a real name lookup cache, this optimization here is useful as it is quicker - * than even a cache lookup). +/* This function implements the main reception loop. Depending on the environment, + * we either use the traditional (but slower) select() or the Linux-specific epoll() + * interface. ./configure settings control which one is used. + * rgerhards, 2009-09-09 */ -BEGINrunInput +#if defined(HAVE_EPOLL_CREATE1) || defined(HAVE_EPOLL_CREATE) +#define NUM_EPOLL_EVENTS 10 +rsRetVal rcvMainLoop(thrdInfo_t *pThrd) +{ + DEFiRet; + int nfds; + int efd; + int i; + struct sockaddr_storage frominetPrev; + int bIsPermitted; + struct epoll_event *udpEPollEvt = NULL; + struct epoll_event currEvt[NUM_EPOLL_EVENTS]; + char errStr[1024]; + + /* start "name caching" algo by making sure the previous system indicator + * is invalidated. + */ + bIsPermitted = 0; + memset(&frominetPrev, 0, sizeof(frominetPrev)); + + CHKmalloc(udpEPollEvt = calloc(udpLstnSocks[0], sizeof(struct epoll_event))); + +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imudp uses epoll_create1()\n"); + efd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("imudp uses epoll_create()\n"); + efd = epoll_create(NUM_EPOLL_EVENTS); +# endif + if(efd < 0) { + DBGPRINTF("epoll_create1() could not create fd\n"); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } + + /* fill the epoll set - we need to do this only once, as the set + * can not change dyamically. + */ + for (i = 0; i < *udpLstnSocks; i++) { + if (udpLstnSocks[i+1] != -1) { + udpEPollEvt[i].events = EPOLLIN | EPOLLET; + udpEPollEvt[i].data.u64 = i+1; + if(epoll_ctl(efd, EPOLL_CTL_ADD, udpLstnSocks[i+1], &(udpEPollEvt[i])) < 0) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + errmsg.LogError(errno, NO_ERRCODE, "epoll_ctrl failed on fd %d with %s\n", + udpLstnSocks[i+1], errStr); + } + } + } + + while(1) { + /* wait for io to become ready */ + nfds = epoll_wait(efd, currEvt, NUM_EPOLL_EVENTS, -1); + DBGPRINTF("imudp: epoll_wait() returned with %d fds\n", nfds); + + if(pThrd->bShallStop == TRUE) + break; /* terminate input! */ + + for(i = 0 ; i < nfds ; ++i) { + processSocket(pThrd, udpLstnSocks[currEvt[i].data.u64], &frominetPrev, &bIsPermitted, + udpRulesets[currEvt[i].data.u64]); + } + } + +finalize_it: + if(udpEPollEvt != NULL) + free(udpEPollEvt); + + RETiRet; +} +#else /* #if HAVE_EPOLL_CREATE1 */ +/* this is the code for the select() interface */ +rsRetVal rcvMainLoop(thrdInfo_t *pThrd) +{ + DEFiRet; int maxfds; int nfds; int i; fd_set readfds; struct sockaddr_storage frominetPrev; int bIsPermitted; - uchar fromHost[NI_MAXHOST]; - uchar fromHostIP[NI_MAXHOST]; - uchar fromHostFQDN[NI_MAXHOST]; -CODESTARTrunInput + /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); - /* this is an endless loop - it is terminated when the thread is - * signalled to do so. This, however, is handled by the framework, - * right into the sleep below. - */ + DBGPRINTF("imudp uses select()\n"); + while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -277,7 +396,7 @@ CODESTARTrunInput * is given without -a, we do not need to listen at all.. */ maxfds = 0; - FD_ZERO (&readfds); + FD_ZERO(&readfds); /* Add the UDP listen sockets to the list of read descriptors. */ for (i = 0; i < *udpLstnSocks; i++) { @@ -291,32 +410,50 @@ CODESTARTrunInput if(Debug) { dbgprintf("--------imUDP calling select, active file descriptors (max %d): ", maxfds); for (nfds = 0; nfds <= maxfds; ++nfds) - if ( FD_ISSET(nfds, &readfds) ) + if(FD_ISSET(nfds, &readfds)) dbgprintf("%d ", nfds); dbgprintf("\n"); } /* wait for io to become ready */ nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ for(i = 0; nfds && i < *udpLstnSocks; i++) { if(FD_ISSET(udpLstnSocks[i+1], &readfds)) { - processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, - fromHost, fromHostFQDN, fromHostIP); + processSocket(pThrd, udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, + udpRulesets[i+1]); --nfds; /* indicate we have processed one descriptor */ } } /* end of a run, back to loop for next recv() */ } - return iRet; + RETiRet; +} +#endif /* #if HAVE_EPOLL_CREATE1 */ + +/* This function is called to gather input. + * Note that udpLstnSocks must be non-NULL because otherwise we would not have + * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 + */ +BEGINrunInput +CODESTARTrunInput + iRet = rcvMainLoop(pThrd); ENDrunInput /* initialize and return if will run or not */ BEGINwillRun CODESTARTwillRun + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imudp"), sizeof("imudp") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + net.PrintAllowedSenders(1); /* UDP */ + net.HasRestrictions(UCHAR_CONSTANT("UDP"), &bDoACLCheck); /* UDP */ /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) @@ -324,9 +461,7 @@ CODESTARTwillRun iMaxLine = glbl.GetMaxLine(); - if((pRcvBuf = malloc((iMaxLine + 1) * sizeof(char))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pRcvBuf = MALLOC((iMaxLine + 1) * sizeof(char))); finalize_it: ENDwillRun @@ -338,11 +473,15 @@ CODESTARTafterRun if(udpLstnSocks != NULL) { net.closeUDPListenSockets(udpLstnSocks); udpLstnSocks = NULL; + free(udpRulesets); + udpRulesets = NULL; } if(pRcvBuf != NULL) { free(pRcvBuf); pRcvBuf = NULL; } + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -352,13 +491,23 @@ CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); ENDmodExit +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) @@ -367,10 +516,6 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a free(pszBindAddr); pszBindAddr = NULL; } - if(udpLstnSocks != NULL) { - net.closeUDPListenSockets(udpLstnSocks); - udpLstnSocks = NULL; - } iTimeRequery = TIME_REQUERY_DFLT;/* the default is to query only every second time */ return RS_RET_OK; } @@ -383,9 +528,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputudpserverbindruleset", 0, eCmdHdlrGetWord, + setRuleset, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserverrun", 0, eCmdHdlrGetWord, addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 1d88a2b5..8ad2b9e9 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-20 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -37,12 +37,16 @@ #include <sys/un.h> #include "dirty.h" #include "cfsysline.h" +#include "unicode-helper.h" #include "module-template.h" #include "srUtils.h" #include "errmsg.h" #include "net.h" #include "glbl.h" #include "msg.h" +#include "prop.h" +#include "debug.h" +#include "unlimited_select.h" MODULE_TYPE_INPUT @@ -66,15 +70,19 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) +static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ +static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ static int startIndexUxLocalSockets; /* process funix from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup */ static int funixParseHost[MAXFUNIX] = { 0, }; /* should parser parse host name? read-only after startup */ static int funixFlags[MAXFUNIX] = { IGNDATE, }; /* should parser parse host name? read-only after startup */ +static int funixCreateSockPath[MAXFUNIX] = { 0, }; /* auto-creation of socket directory? */ static uchar *funixn[MAXFUNIX] = { (uchar*) _PATH_LOG }; /* read-only after startup */ -static uchar *funixHName[MAXFUNIX] = { NULL, }; /* host-name override - if set, use this instead of actual name */ +static prop_t *funixHName[MAXFUNIX] = { NULL, }; /* host-name override - if set, use this instead of actual name */ static int funixFlowCtl[MAXFUNIX] = { eFLOWCTL_NO_DELAY, }; /* flow control settings for this socket */ static int funix[MAXFUNIX] = { -1, }; /* read-only after startup */ static int nfunix = 1; /* number of Unix sockets open / read-only after startup */ @@ -85,6 +93,8 @@ static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +#define DFLT_bCreateSockPath 0 +static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? */ /* set the timestamp ignore / not ignore option for the system @@ -115,29 +125,41 @@ static rsRetVal setSystemLogFlowControl(void __attribute__((unused)) *pVal, int * rgerhards, 2007-12-20 * added capability to specify hostname for socket -- rgerhards, 2008-08-01 */ -static rsRetVal addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) +static rsRetVal +addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) { + DEFiRet; + if(nfunix < MAXFUNIX) { if(*pNewVal == ':') { funixParseHost[nfunix] = 1; - } - else { + } else { funixParseHost[nfunix] = 0; } - funixHName[nfunix] = pLogHostName; - pLogHostName = NULL; /* re-init for next, not freed because funixHName[] now owns it */ + CHKiRet(prop.Construct(&(funixHName[nfunix]))); + if(pLogHostName == NULL) { + CHKiRet(prop.SetString(funixHName[nfunix], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + } else { + CHKiRet(prop.SetString(funixHName[nfunix], pLogHostName, ustrlen(pLogHostName))); + /* reset hostname for next socket */ + free(pLogHostName); + pLogHostName = NULL; + } + CHKiRet(prop.ConstructFinalize(funixHName[nfunix])); funixFlowCtl[nfunix] = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; funixFlags[nfunix] = bIgnoreTimestamp ? IGNDATE : NOFLAG; + funixCreateSockPath[nfunix] = bCreateSockPath; funixn[nfunix++] = pNewVal; - } - else { + } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", pNewVal); } - return RS_RET_OK; +finalize_it: + RETiRet; } + /* free the funixn[] socket names - needed as cleanup on several places * note that nfunix is NOT reset! funixn[0] is never freed, as it comes from * the constant memory pool - and if not, it is freeed via some other pointer. @@ -152,8 +174,7 @@ static rsRetVal discardFunixn(void) funixn[i] = NULL; } if(funixHName[i] != NULL) { - free(funixHName[i]); - funixHName[i] = NULL; + prop.Destruct(&(funixHName[i])); } } @@ -161,7 +182,7 @@ static rsRetVal discardFunixn(void) } -static int create_unix_socket(const char *path) +static int create_unix_socket(const char *path, int bCreatePath) { struct sockaddr_un sunx; int fd; @@ -173,6 +194,9 @@ static int create_unix_socket(const char *path) memset(&sunx, 0, sizeof(sunx)); sunx.sun_family = AF_UNIX; + if(bCreatePath) { + makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); + } (void) strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); fd = socket(AF_UNIX, SOCK_DGRAM, 0); if (fd < 0 || bind(fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || @@ -186,6 +210,35 @@ static int create_unix_socket(const char *path) } +/* submit received message to the queue engine + */ +static inline rsRetVal +SubmitMsg(uchar *pRcv, int lenRcv, int iSock) +{ + msg_t *pMsg; + DEFiRet; + + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstruct(&pMsg)); + MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); + MsgSetInputName(pMsg, pInputName); + MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + + if(funixParseHost[iSock]) { + pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; + } else { + pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING; + } + + MsgSetRcvFrom(pMsg, funixHName[iSock]); + CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); + CHKiRet(submitMsg(pMsg)); + +finalize_it: + RETiRet; +} + + /* This function receives data from a socket indicated to be ready * to receive and submits the message received for processing. * rgerhards, 2007-12-20 @@ -214,16 +267,13 @@ static rsRetVal readSocket(int fd, int iSock) if((size_t) iMaxLine < sizeof(bufRcv) - 1) { pRcv = bufRcv; } else { - CHKmalloc(pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))); + CHKmalloc(pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))); } iRcvd = recv(fd, pRcv, iMaxLine, 0); dbgprintf("Message from UNIX socket: #%d\n", fd); if (iRcvd > 0) { - parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], - (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock] ? (funixFlags[iSock] | PARSE_HOSTNAME) : funixFlags[iSock], - funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -245,7 +295,13 @@ BEGINrunInput int nfds; int i; int fd; - fd_set readfds; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = malloc(glbl.GetFdSetSize()); +#else + fd_set readfds; + fd_set *pReadfds = &readfds; +#endif + CODESTARTrunInput /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, @@ -259,11 +315,11 @@ CODESTARTrunInput * is given without -a, we do not need to listen at all.. */ maxfds = 0; - FD_ZERO (&readfds); + FD_ZERO (pReadfds); /* Copy master connections */ for (i = startIndexUxLocalSockets; i < nfunix; i++) { if (funix[i] != -1) { - FD_SET(funix[i], &readfds); + FD_SET(funix[i], pReadfds); if (funix[i]>maxfds) maxfds=funix[i]; } } @@ -271,22 +327,28 @@ CODESTARTrunInput if(Debug) { dbgprintf("--------imuxsock calling select, active file descriptors (max %d): ", maxfds); for (nfds= 0; nfds <= maxfds; ++nfds) - if ( FD_ISSET(nfds, &readfds) ) + if ( FD_ISSET(nfds, pReadfds) ) dbgprintf("%d ", nfds); dbgprintf("\n"); } /* wait for io to become ready */ - nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); + nfds = select(maxfds+1, (fd_set *) pReadfds, NULL, NULL, NULL); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ for (i = 0; i < nfunix && nfds > 0; i++) { - if ((fd = funix[i]) != -1 && FD_ISSET(fd, &readfds)) { + if(glbl.GetGlobalInputTermState() == 1) + ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ + if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) { readSocket(fd, i); --nfds; /* indicate we have processed one */ } } } +finalize_it: + freeFdSet(pReadfds); RETiRet; ENDrunInput @@ -302,11 +364,16 @@ CODESTARTwillRun /* initialize and return if will run or not */ for (i = startIndexUxLocalSockets ; i < nfunix ; i++) { - if ((funix[i] = create_unix_socket((char*) funixn[i])) != -1) + if ((funix[i] = create_unix_socket((char*) funixn[i], funixCreateSockPath[i])) != -1) dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); } - RETiRet; + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + +finalize_it: ENDwillRun @@ -324,14 +391,14 @@ CODESTARTafterRun if (funixn[i] && funix[i] != -1) unlink((char*) funixn[i]); /* free no longer needed string */ - if(pLogSockName != NULL) - free(pLogSockName); - if(pLogHostName != NULL) { - free(pLogHostName); - } + free(pLogSockName); + free(pLogHostName); discardFunixn(); nfunix = 1; + + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -339,12 +406,21 @@ BEGINmodExit CODESTARTmodExit objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDmodExit +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) @@ -363,6 +439,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a nfunix = 1; bIgnoreTimestamp = 1; bUseFlowCtl = 0; + bCreateSockPath = DFLT_bCreateSockPath; return RS_RET_OK; } @@ -375,6 +452,7 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); @@ -384,6 +462,15 @@ CODEmodInit_QueryRegCFSLineHdlr funix[i] = -1; } + CHKiRet(prop.Construct(&pLocalHostIP)); + CHKiRet(prop.SetString(pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pLocalHostIP)); + + /* now init listen socket zero, the local log socket */ + CHKiRet(prop.Construct(&(funixHName[0]))); + CHKiRet(prop.SetString(funixHName[0], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.ConstructFinalize(funixHName[0])); + /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"omitlocallogging", 0, eCmdHdlrBinary, NULL, &bOmitLocalLogging, STD_LOADABLE_MODULE_ID)); @@ -395,6 +482,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &pLogHostName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketflowcontrol", 0, eCmdHdlrBinary, NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, + NULL, &bCreateSockPath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, diff --git a/plugins/omdbalerting/Makefile.am b/plugins/omdbalerting/Makefile.am new file mode 100644 index 00000000..becf29b0 --- /dev/null +++ b/plugins/omdbalerting/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omdbalerting.la + +omdbalerting_la_SOURCES = omdbalerting.c +omdbalerting_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omdbalerting_la_LDFLAGS = -module -avoid-version +omdbalerting_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omdbalerting/omdbalerting.c b/plugins/omdbalerting/omdbalerting.c new file mode 100644 index 00000000..2e04391c --- /dev/null +++ b/plugins/omdbalerting/omdbalerting.c @@ -0,0 +1,144 @@ +/* omdbalerting.c + * generate alerts based on database contents - so far a skeleton + * left for implementation by somebody else (skeleton created on request). + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-11-17 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* config variables */ + + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omdbalerting:", sizeof(":dbalerting:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omdbalerting:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + /* we request the standard interface via template, others may be more useful + * here. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + // SAMPLE! CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomdbalertingensurelfending", 0, eCmdHdlrBinary, NULL, + // &bEnsureLFEnding, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 361f657f..08aaad80 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -44,6 +44,7 @@ #include <pthread.h> #include <gssapi/gssapi.h> #include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "net.h" @@ -192,7 +193,7 @@ static rsRetVal TCPSendGSSInit(void *pvData) base = (gss_base_service_name == NULL) ? "host" : gss_base_service_name; out_tok.length = strlen(pData->f_hname) + strlen(base) + 2; - CHKmalloc(out_tok.value = malloc(out_tok.length)); + CHKmalloc(out_tok.value = MALLOC(out_tok.length)); strcpy(out_tok.value, base); strcat(out_tok.value, "@"); strcat(out_tok.value, pData->f_hname); @@ -408,13 +409,13 @@ CODESTARTdoAction * hard-coded but this may be changed to a config parameter. * rgerhards, 2006-11-30 */ - if(pData->compressionLevel && (l > MIN_SIZE_FOR_COMPRESS)) { + if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { Bytef *out; uLongf destLen = sizeof(out) / sizeof(Bytef); uLong srcLen = l; int ret; /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ - CHKmalloc(out = (Bytef*) malloc(iMaxLine + iMaxLine/100 + 12)); + CHKmalloc(out = (Bytef*) MALLOC(iMaxLine + iMaxLine/100 + 12)); out[0] = 'z'; out[1] = '\0'; ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, @@ -430,11 +431,14 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ dbgprintf("Compression failed, sending uncompressed message\n"); + free(out); } else if(destLen+1 < l) { /* only use compression if there is a gain in using it! */ dbgprintf("there is gain in compression, so we do it\n"); psz = (char*) out; l = destLen + 1; /* take care for the "z" at message start! */ + } else { + free(out); } ++destLen; } @@ -449,6 +453,12 @@ CODESTARTdoAction break; } finalize_it: +# ifdef USE_NETZIP + if(psz != (char*) ppString[0]) { + /* we need to free temporary buffer, alloced above - Naoya Nakazawa, 2010-01-11 */ + free(psz); + } +# endif ENDdoAction @@ -553,7 +563,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) tmp = ++p; for(i=0 ; *p && isdigit((int) *p) ; ++p, ++i) /* SKIP AND COUNT */; - pData->port = malloc(i + 1); + pData->port = MALLOC(i + 1); if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store syslog forwarding port, " "using default port, results may not be what you intend\n"); diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index 5faadce3..324e1a77 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -44,12 +44,13 @@ #include <netdb.h> #include <time.h> #include <sys/socket.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "datetime.h" #include "glbl.h" MODULE_TYPE_OUTPUT @@ -59,6 +60,7 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) +DEFobjCurrIf(datetime) /* we add a little support for multiple recipients. We do this via a * singly-linked list, enqueued from the top. -- rgerhards, 2008-08-04 @@ -478,7 +480,7 @@ mkSMTPTimestamp(uchar *pszBuf, size_t lenBuf) static const char szDay[][4] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; static const char szMonth[][4] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; - time(&tCurr); + datetime.GetTime(&tCurr); gmtime_r(&tCurr, &tmCurr); snprintf((char*)pszBuf, lenBuf, "Date: %s, %2d %s %4d %2d:%02d:%02d UT\r\n", szDay[tmCurr.tm_wday], tmCurr.tm_mday, szMonth[tmCurr.tm_mon], 1900 + tmCurr.tm_year, tmCurr.tm_hour, tmCurr.tm_min, tmCurr.tm_sec); @@ -669,6 +671,7 @@ CODESTARTmodExit freeConfigVariables(); /* release what we no longer need */ + objRelease(datetime, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -698,6 +701,7 @@ CODEmodInit_QueryRegCFSLineHdlr /* tell which objects we need */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); dbgprintf("ommail version %s initializing\n", VERSION); diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index ecf738a9..d6870a7b 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -36,7 +36,7 @@ #include <errno.h> #include <time.h> #include <mysql.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 331b7dd4..48ee1fa4 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -47,9 +47,9 @@ $OmoracleStatement \ insert into foo(hostname,message)values(:host,:message) - Also note that identifiers to placeholders are arbitrarry. You - need to define the properties on the template in the correct order - you want them passed to the statement! + Also note that identifiers to placeholders are arbitrary. You need + to define the properties on the template in the correct order you + want them passed to the statement! This file is licensed under the terms of the GPL version 3 or, at your choice, any later version. Exceptionally (perhaps), you are @@ -87,7 +87,8 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) -/** */ +/** Structure defining a batch of items to be sent to the database in + * the same statement execution. */ struct oracle_batch { /* Batch size */ @@ -162,8 +163,10 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_SUCCESS; break; case OCI_SUCCESS_WITH_INFO: - errmsg.LogError(0, NO_ERRCODE, "OCI SUCCESS - With info\n"); - break; + OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); + errmsg.LogError(0, NO_ERRCODE, "OCI SUCCESS - With info: %s", + buf); + return OCI_SUCCESS_WITH_INFO; case OCI_NEED_DATA: errmsg.LogError(0, NO_ERRCODE, "OCI NEEDS MORE DATA\n"); break; @@ -180,6 +183,9 @@ static int oci_errors(void* handle, ub4 htype, sword status) break; case OCI_INVALID_HANDLE: errmsg.LogError(0, NO_ERRCODE, "OCI INVALID HANDLE\n"); + /* In this case we may have to trigger a call to + * tryResume(). */ + return RS_RET_SUSPENDED; break; case OCI_STILL_EXECUTING: errmsg.LogError(0, NO_ERRCODE, "Still executing...\n"); @@ -332,6 +338,48 @@ CODESTARTcreateInstance finalize_it: ENDcreateInstance +/* Analyses the errors during a batch statement execution, and logs + * all the corresponding ORA-MESSAGES, together with some useful + * information. */ +static void log_detailed_err(instanceData* pData) +{ + DEFiRet; + int errs, i, row, code, j; + OCIError *er = NULL, *er2 = NULL; + unsigned char buf[MAX_BUFSIZE]; + + OCIAttrGet(pData->statement, OCI_HTYPE_STMT, &errs, 0, + OCI_ATTR_NUM_DML_ERRORS, pData->error); + errmsg.LogError(0, NO_ERRCODE, "OCI: %d errors in execution of " + "statement: %s", errs, pData->txt_statement); + + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &er, OCI_HTYPE_ERROR, + 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &er2, OCI_HTYPE_ERROR, + 0, NULL)); + + for (i = 0; i < errs; i++) { + OCIParamGet(pData->error, OCI_HTYPE_ERROR, + er2, &er, i); + OCIAttrGet(er, OCI_HTYPE_ERROR, &row, 0, + OCI_ATTR_DML_ROW_OFFSET, er2); + errmsg.LogError(0, NO_ERRCODE, "OCI failure in row %d:", row); + for (j = 0; j < pData->batch.arguments; j++) + errmsg.LogError(0, NO_ERRCODE, "%s", + pData->batch.parameters[j][row]); + OCIErrorGet(er, 1, NULL, &code, buf, sizeof buf, + OCI_HTYPE_ERROR); + errmsg.LogError(0, NO_ERRCODE, "FAILURE DETAILS: %s", buf); + } + +finalize_it: + OCIHandleFree(er, OCI_HTYPE_ERROR); + OCIHandleFree(er2, OCI_HTYPE_ERROR); +} + + /* Inserts all stored statements into the database, releasing any * allocated memory. */ static int insert_to_db(instanceData* pData) @@ -346,6 +394,10 @@ static int insert_to_db(instanceData* pData) OCI_BATCH_ERRORS)); finalize_it: + if (iRet == OCI_SUCCESS_WITH_INFO) { + log_detailed_err(pData); + iRet = RS_RET_OK; + } pData->batch.n = 0; OCITransCommit(pData->service, pData->error, 0); dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? diff --git a/plugins/ompgsql/createDB.sql b/plugins/ompgsql/createDB.sql index facdcaa0..2f72a0a6 100644 --- a/plugins/ompgsql/createDB.sql +++ b/plugins/ompgsql/createDB.sql @@ -1,4 +1,4 @@ -CREATE DATABASE "Syslog"; +CREATE DATABASE 'Syslog' WITH ENCODING 'SQL_ASCII'; \c Syslog; CREATE TABLE SystemEvents ( diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 6daac1c7..c13f58e9 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -40,7 +40,7 @@ #include <errno.h> #include <time.h> #include <libpq-fe.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" @@ -65,6 +65,8 @@ typedef struct _instanceData { } instanceData; +static rsRetVal writePgSQL(uchar *psz, instanceData *pData); + BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance @@ -189,7 +191,8 @@ tryExec(uchar *pszCmd, instanceData *pData) * a sql format error - connection aborts were properly handled * before my patch. -- rgerhards, 2009-04-17 */ -rsRetVal writePgSQL(uchar *psz, instanceData *pData) +static rsRetVal +writePgSQL(uchar *psz, instanceData *pData) { int bHadError = 0; DEFiRet; @@ -227,16 +230,44 @@ BEGINtryResume CODESTARTtryResume if(pData->f_hpgsql == NULL) { iRet = initPgSQL(pData, 1); + if(iRet == RS_RET_OK) { + /* the code above seems not to actually connect to the database. As such, we do a + * dummy statement (a pointless select...) to verify the connection and return + * success only when that statemetn succeeds. Note that I am far from being a + * PostgreSQL expert, so any patch that does the desired result in a more + * intelligent way is highly welcome. -- rgerhards, 2009-12-16 + */ + iRet = writePgSQL((uchar*)"select 'a' as a", pData); + } + } ENDtryResume + +BEGINbeginTransaction +CODESTARTbeginTransaction +dbgprintf("ompgsql: beginTransaction\n"); + iRet = writePgSQL((uchar*) "begin", pData); /* TODO: make user-configurable */ +ENDbeginTransaction + + BEGINdoAction CODESTARTdoAction dbgprintf("\n"); - iRet = writePgSQL(ppString[0], pData); + CHKiRet(writePgSQL(ppString[0], pData)); + if(bCoreSupportsBatching) + iRet = RS_RET_DEFER_COMMIT; +finalize_it: ENDdoAction +BEGINendTransaction +CODESTARTendTransaction + iRet = writePgSQL((uchar*) "commit;", pData); /* TODO: make user-configurable */ +dbgprintf("ompgsql: endTransaction\n"); +ENDendTransaction + + BEGINparseSelectorAct int iPgSQLPropErr = 0; CODESTARTparseSelectorAct @@ -314,6 +345,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ ENDqueryEtryPt @@ -322,6 +354,8 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); + INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); + DBGPRINTF("ompgsql: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not "); ENDmodInit /* vi:set ai: */ diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 2a078a6d..01fa7cea 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -36,7 +36,7 @@ #include <errno.h> #include <unistd.h> #include <wait.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 8d74c82f..349e45aa 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -36,13 +36,14 @@ #include <errno.h> #include <ctype.h> #include <librelp.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "cfsysline.h" #include "module-template.h" #include "glbl.h" #include "errmsg.h" +#include "debug.h" MODULE_TYPE_OUTPUT @@ -260,7 +261,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) tmp = ++p; for(i=0 ; *p && isdigit((int) *p) ; ++p, ++i) /* SKIP AND COUNT */; - pData->port = malloc(i + 1); + pData->port = MALLOC(i + 1); if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store relp port, " "using default port, results may not be what you intend\n"); diff --git a/plugins/omruleset/Makefile.am b/plugins/omruleset/Makefile.am new file mode 100644 index 00000000..fdd91a6e --- /dev/null +++ b/plugins/omruleset/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omruleset.la + +omruleset_la_SOURCES = omruleset.c +omruleset_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omruleset_la_LDFLAGS = -module -avoid-version +omruleset_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omruleset/omruleset.c b/plugins/omruleset/omruleset.c new file mode 100644 index 00000000..0e0fc13b --- /dev/null +++ b/plugins/omruleset/omruleset.c @@ -0,0 +1,231 @@ +/* omruleset.c + * This is a very special output module. It permits to pass a message object + * to another rule set. While this is a very simple action, it enables very + * complex configurations, e.g. it supports high-speed "and" conditions, sending + * data to the same file in a non-racy way, include functionality as well as + * some high-performance optimizations (in case the rule sets have the necessary + * queue definitions). So while this code is small, it is pretty important. + * + * NOTE: read comments in module-template.h for details on the calling interface! + * + * File begun on 2009-11-02 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "ruleset.h" +#include "cfsysline.h" +#include "dirty.h" + +MODULE_TYPE_OUTPUT + +/* static data */ +DEFobjCurrIf(ruleset); +DEFobjCurrIf(errmsg); + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* config variables */ +ruleset_t *pRuleset = NULL; /* ruleset to enqueue message to (NULL = Default, not recommended) */ +uchar *pszRulesetName = NULL; + + +typedef struct _instanceData { + ruleset_t *pRuleset; /* ruleset to enqueue to */ + uchar *pszRulesetName; /* primarily for debugging/display purposes */ +} instanceData; + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance + free(pData->pszRulesetName); +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + dbgprintf("omruleset target %s[%p]\n", (char*) pData->pszRulesetName, pData->pRuleset); +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +/* Note that we change the flow control type to "no delay", because at this point in + * rsyslog procesing we can not really slow down the producer any longer, as we already + * work off a queue. So a delay would just block out execution for longer than needed. + */ +BEGINdoAction + msg_t *pMsg; +CODESTARTdoAction + CHKmalloc(pMsg = MsgDup((msg_t*) ppString[0])); + DBGPRINTF(":omruleset: forwarding message %p to ruleset %s[%p]\n", pMsg, + (char*) pData->pszRulesetName, pData->pRuleset); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + MsgSetRuleset(pMsg, pData->pRuleset); + submitMsg(pMsg); +finalize_it: +ENDdoAction + +/* set the ruleset name */ +static rsRetVal +setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, RS_RET_RULESET_NOT_FOUND, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pszRulesetName = pszName; /* save for later display purposes */ + +finalize_it: + if(iRet != RS_RET_OK) { /* cleanup needed? */ + free(pszName); + } + RETiRet; +} + + +BEGINparseSelectorAct + int iTplOpts; +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omruleset:", sizeof(":omruleset:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + if(pRuleset == NULL) { + errmsg.LogError(0, RS_RET_NO_RULESET, "error: no ruleset was specified, use " + "$ActionOmrulesetRulesetName directive first!"); + ABORT_FINALIZE(RS_RET_NO_RULESET); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omruleset:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + iTplOpts = OMSR_TPL_AS_MSG; + /* we call the message below because we need to call it via our interface definition. However, + * the format specified (if any) is always ignored. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, iTplOpts, (uchar*) "RSYSLOG_FileFormat")); + pData->pRuleset = pRuleset; + pData->pszRulesetName = pszRulesetName; + pRuleset = NULL; /* re-set, because there is a high risk of unwanted behavior if we leave it in! */ + pszRulesetName = NULL; /* note: we must not free, as we handed over this pointer to the instanceDat to the instanceDataa! */ +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit + free(pszRulesetName); + objRelease(errmsg, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + pRuleset = NULL; + RETiRet; +} + + +BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bMsgPassingSupported; /* does core support template passing as an array? */ +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + /* check if the rsyslog core supports parameter passing code */ + bMsgPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + /* found entry point, so let's see if core supports msg passing */ + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_MSG) + bMsgPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */ + } + + if(!bMsgPassingSupported) { + DBGPRINTF("omruleset: msg-passing is not supported by rsyslog core, can not continue.\n"); + ABORT_FINALIZE(RS_RET_NO_MSG_PASSING); + } + + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomrulesetrulesetname", 0, eCmdHdlrGetWord, + setRuleset, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/plugins/omsnmp/omsnmp.c b/plugins/omsnmp/omsnmp.c index 72fa8d64..b973b09d 100644 --- a/plugins/omsnmp/omsnmp.c +++ b/plugins/omsnmp/omsnmp.c @@ -36,7 +36,7 @@ #include <netdb.h> #include <ctype.h> #include <assert.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "cfsysline.h" #include "module-template.h" @@ -222,7 +222,7 @@ static rsRetVal omsnmp_sendsnmp(instanceData *pData, uchar *psz) ABORT_FINALIZE(RS_RET_DISABLE_ACTION); } - pdu->enterprise = (oid *) malloc(enterpriseoidlen * sizeof(oid)); + pdu->enterprise = (oid *) MALLOC(enterpriseoidlen * sizeof(oid)); memcpy(pdu->enterprise, enterpriseoid, enterpriseoidlen * sizeof(oid)); pdu->enterprise_length = enterpriseoidlen; diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 181895a4..b3ec6287 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -35,7 +35,7 @@ #include <signal.h> #include <errno.h> #include <unistd.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" @@ -50,11 +50,13 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA /* config variables */ -static int bUseArrayInterface; /* shall action use array instead of string template interface? */ +static int bUseArrayInterface = 0; /* shall action use array instead of string template interface? */ +static int bEnsureLFEnding = 1; /* shall action use array instead of string template interface? */ typedef struct _instanceData { int bUseArrayInterface; /* uses action use array instead of string template interface? */ + int bEnsureLFEnding; /* ensure that a linefeed is written at the end of EACH record (test aid for nettester) */ } instanceData; BEGINcreateInstance @@ -90,6 +92,7 @@ BEGINdoAction int iParam; int iBuf; char szBuf[65564]; + size_t len; CODESTARTdoAction if(pData->bUseArrayInterface) { /* if we use array passing, we need to put together a string @@ -120,7 +123,11 @@ CODESTARTdoAction } else { toWrite = (char*) ppString[0]; } - write(1, toWrite, strlen(toWrite)); /* 1 is stdout! */ + len = strlen(toWrite); + write(1, toWrite, len); /* 1 is stdout! */ + if(pData->bEnsureLFEnding && toWrite[len-1] != '\n') { + write(1, "\n", 1); /* write missing LF */ + } ENDdoAction @@ -143,6 +150,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) iTplOpts = (bUseArrayInterface == 0) ? 0 : OMSR_TPL_AS_ARRAY; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, iTplOpts, (uchar*) "RSYSLOG_FileFormat")); pData->bUseArrayInterface = bUseArrayInterface; + pData->bEnsureLFEnding = bEnsureLFEnding; CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -165,6 +173,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a { DEFiRet; bUseArrayInterface = 0; + bEnsureLFEnding = 1; RETiRet; } @@ -195,6 +204,8 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomstdoutarrayinterface", 0, eCmdHdlrBinary, NULL, &bUseArrayInterface, STD_LOADABLE_MODULE_ID)); } + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomstdoutensurelfending", 0, eCmdHdlrBinary, NULL, + &bEnsureLFEnding, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/omtemplate/omtemplate.c b/plugins/omtemplate/omtemplate.c index e35968ad..5577f8c6 100644 --- a/plugins/omtemplate/omtemplate.c +++ b/plugins/omtemplate/omtemplate.c @@ -36,7 +36,7 @@ #include <signal.h> #include <errno.h> #include <time.h> -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omtesting/omtesting.c b/plugins/omtesting/omtesting.c index 411bcf88..9442f691 100644 --- a/plugins/omtesting/omtesting.c +++ b/plugins/omtesting/omtesting.c @@ -22,7 +22,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -46,12 +46,15 @@ #include <stdio.h> #include <stdarg.h> #include <stdlib.h> +#include <time.h> #include <string.h> #include <ctype.h> #include <assert.h> #include "dirty.h" #include "syslogd-types.h" #include "module-template.h" +#include "conf.h" +#include "cfsysline.h" MODULE_TYPE_OUTPUT @@ -59,9 +62,18 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA +static int bEchoStdout = 0; /* echo non-failed messages to stdout */ + typedef struct _instanceData { + enum { MD_SLEEP, MD_FAIL, MD_RANDFAIL, MD_ALWAYS_SUSPEND } + mode; + int bEchoStdout; int iWaitSeconds; int iWaitUSeconds; /* milli-seconds (one million of a second, just to make sure...) */ + int iCurrCallNbr; + int iFailFrequency; + int iResumeAfter; + int iCurrRetries; } instanceData; BEGINcreateInstance @@ -85,19 +97,106 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature -BEGINtryResume -CODESTARTtryResume -ENDtryResume +/* implement "fail" command in retry processing */ +static rsRetVal doFailOnResume(instanceData *pData) +{ + DEFiRet; -BEGINdoAction -CODESTARTdoAction + dbgprintf("fail retry curr %d, max %d\n", pData->iCurrRetries, pData->iResumeAfter); + if(++pData->iCurrRetries == pData->iResumeAfter) { + iRet = RS_RET_OK; + } else { + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +/* implement "fail" command */ +static rsRetVal doFail(instanceData *pData) +{ + DEFiRet; + + dbgprintf("fail curr %d, frquency %d\n", pData->iCurrCallNbr, pData->iFailFrequency); + if(pData->iCurrCallNbr++ % pData->iFailFrequency == 0) { + pData->iCurrRetries = 0; + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +/* implement "sleep" command */ +static rsRetVal doSleep(instanceData *pData) +{ + DEFiRet; struct timeval tvSelectTimeout; dbgprintf("sleep(%d, %d)\n", pData->iWaitSeconds, pData->iWaitUSeconds); tvSelectTimeout.tv_sec = pData->iWaitSeconds; tvSelectTimeout.tv_usec = pData->iWaitUSeconds; /* milli seconds */ select(0, NULL, NULL, NULL, &tvSelectTimeout); - //dbgprintf(":omtesting: end doAction(), iRet %d\n", iRet); + RETiRet; +} + + +/* implement "randomfail" command */ +static rsRetVal doRandFail(void) +{ + DEFiRet; + if((rand() >> 4) < (RAND_MAX >> 5)) { /* rougly same probability */ + iRet = RS_RET_OK; + dbgprintf("omtesting randfail: succeeded this time\n"); + } else { + iRet = RS_RET_SUSPENDED; + dbgprintf("omtesting randfail: failed this time\n"); + } + RETiRet; +} + + +BEGINtryResume +CODESTARTtryResume + dbgprintf("omtesting tryResume() called\n"); + switch(pData->mode) { + case MD_SLEEP: + break; + case MD_FAIL: + iRet = doFailOnResume(pData); + break; + case MD_RANDFAIL: + iRet = doRandFail(); + break; + case MD_ALWAYS_SUSPEND: + iRet = RS_RET_SUSPENDED; + } + dbgprintf("omtesting tryResume() returns iRet %d\n", iRet); +ENDtryResume + + +BEGINdoAction +CODESTARTdoAction + dbgprintf("omtesting received msg '%s'\n", ppString[0]); + switch(pData->mode) { + case MD_SLEEP: + iRet = doSleep(pData); + break; + case MD_FAIL: + iRet = doFail(pData); + break; + case MD_RANDFAIL: + iRet = doRandFail(); + case MD_ALWAYS_SUSPEND: + iRet = RS_RET_SUSPENDED; + } + + if(iRet == RS_RET_OK && pData->bEchoStdout) { + fprintf(stdout, "%s", ppString[0]); + fflush(stdout); + } + dbgprintf(":omtesting: end doAction(), iRet %d\n", iRet); ENDdoAction @@ -113,7 +212,7 @@ BEGINparseSelectorAct int i; uchar szBuf[1024]; CODESTARTparseSelectorAct -CODE_STD_STRING_REQUESTparseSelectorAct(0) +CODE_STD_STRING_REQUESTparseSelectorAct(1) /* code here is quick and dirty - if you like, clean it up. But keep * in mind it is just a testing aid ;) -- rgerhards, 2007-12-31 */ @@ -135,6 +234,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(0) if(isspace(*p)) ++p; + dbgprintf("omtesting command: '%s'\n", szBuf); if(!strcmp((char*) szBuf, "sleep")) { /* parse seconds */ for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { @@ -152,12 +252,43 @@ CODE_STD_STRING_REQUESTparseSelectorAct(0) if(isspace(*p)) ++p; pData->iWaitUSeconds = atoi((char*) szBuf); - } - /* once there are other modes, here is the spot to add it! */ - else { + pData->mode = MD_SLEEP; + } else if(!strcmp((char*) szBuf, "fail")) { + /* "fail fail-freqency resume-after" + * fail-frequency specifies how often doAction() fails + * resume-after speicifes how fast tryResume() should come back with success + * all numbers being "times called" + */ + /* parse fail-frequence */ + for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { + szBuf[i] = *p++; + } + szBuf[i] = '\0'; + if(isspace(*p)) + ++p; + pData->iFailFrequency = atoi((char*) szBuf); + /* parse resume-after */ + for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { + szBuf[i] = *p++; + } + szBuf[i] = '\0'; + if(isspace(*p)) + ++p; + pData->iResumeAfter = atoi((char*) szBuf); + pData->iCurrCallNbr = 1; + pData->mode = MD_FAIL; + } else if(!strcmp((char*) szBuf, "randfail")) { + pData->mode = MD_RANDFAIL; + } else if(!strcmp((char*) szBuf, "always_suspend")) { + pData->mode = MD_ALWAYS_SUSPEND; + } else { dbgprintf("invalid mode '%s', doing 'sleep 1 0' - fix your config\n", szBuf); } + pData->bEchoStdout = bEchoStdout; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (uchar*)"RSYSLOG_TraditionalForwardFormat")); + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -177,6 +308,10 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomtestingechostdout", 0, eCmdHdlrBinary, NULL, + &bEchoStdout, STD_LOADABLE_MODULE_ID)); + /* we seed the random-number generator in any case... */ + srand(time(NULL)); ENDmodInit /* * vi:set ai: diff --git a/plugins/omudpspoof/Makefile.am b/plugins/omudpspoof/Makefile.am new file mode 100644 index 00000000..79c495a0 --- /dev/null +++ b/plugins/omudpspoof/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omudpspoof.la + +omudpspoof_la_SOURCES = omudpspoof.c +omudpspoof_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(UDPSPOOF_CFLAGS) +omudpspoof_la_LDFLAGS = -module -avoid-version +omudpspoof_la_LIBADD = $(UDPSPOOF_LIBS) + +EXTRA_DIST = diff --git a/plugins/omudpspoof/omudpspoof.c b/plugins/omudpspoof/omudpspoof.c new file mode 100644 index 00000000..50bc6c9a --- /dev/null +++ b/plugins/omudpspoof/omudpspoof.c @@ -0,0 +1,503 @@ +/* omudpspoof.c + * + * This is a udp-based output module that support spoofing. + * + * This file builds on UDP spoofing code contributed by + * David Lang <david@lang.hm>. I then created a "real" rsyslog module + * out of that code and omfwd. I decided to make it a separate module because + * omfwd already mixes up too many things (TCP & UDP & a differnt modes, + * this has historic reasons), it would not be a good idea to also add + * spoofing to it. And, looking at the requirements, there is little in + * common between omfwd and this module. + * + * Note: I have briefly checked libnet source code and I somewhat have the feeling + * that under some circumstances we may get into trouble with the lib. For + * example, it registers an atexit() handler, which should not play nicely + * with our dynamically loaded modules. Anyhow, I refrain from looking deeper + * at libnet code, especially as testing does not show any real issues. If some + * occur, it may be easier to modify libnet for dynamic load environments than + * using a work-around (as a side not, libnet looks somewhat unmaintained, the CVS + * I can see on sourceforge dates has no updates done less than 7 years ago). + * On the other hand, it looks like libnet is thread safe (at least is appropriately + * compiled, which I hope the standard packages are). So I do not guard calls to + * it with my own mutex calls. + * rgerhards, 2009-07-10 + * + * Copyright 2009 David Lang (spoofing code) + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <netinet/in.h> +#include <netdb.h> +#include <fnmatch.h> +#include <assert.h> +#include <errno.h> +#include <ctype.h> +#include <unistd.h> +#ifdef USE_NETZIP +#include <zlib.h> +#endif +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "net.h" +#include "template.h" +#include "msg.h" +#include "cfsysline.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "dirty.h" +#include "unicode-helper.h" +#include "debug.h" + + +#include <libnet.h> +#define _BSD_SOURCE 1 +#define __BSD_SOURCE 1 +#define __FAVOR_BSD 1 + + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(net) + +typedef struct _instanceData { + uchar *host; + uchar *port; + int *pSockArray; /* sockets to use for UDP */ + int compressionLevel; /* 0 - no compression, else level for zlib */ + struct addrinfo *f_addr; + u_short sourcePort; + u_short sourcePortStart; /* for sorce port iteration */ + u_short sourcePortEnd; +} instanceData; + +#define DFLT_SOURCE_PORT_START 32000 +#define DFLT_SOURCE_PORT_END 42000 + +/* config data */ +static uchar *pszTplName = NULL; /* name of the default template to use */ +static uchar *pszSourceNameTemplate = NULL; /* name of the template containing the spoofing address */ +static uchar *pszTargetHost = NULL; +static uchar *pszTargetPort = NULL; +static int iCompressionLevel = 0; /* zlib compressionlevel, the usual values */ +static int iSourcePortStart = DFLT_SOURCE_PORT_START; +static int iSourcePortEnd = DFLT_SOURCE_PORT_END; + + +/* add some variables needed for libnet */ +libnet_t *libnet_handle; +char errbuf[LIBNET_ERRBUF_SIZE]; + +/* forward definitions */ +static rsRetVal doTryResume(instanceData *pData); + + +/* Close the UDP sockets. + * rgerhards, 2009-05-29 + */ +static rsRetVal +closeUDPSockets(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + if(pData->pSockArray != NULL) { + net.closeUDPListenSockets(pData->pSockArray); + pData->pSockArray = NULL; + freeaddrinfo(pData->f_addr); + pData->f_addr = NULL; + } + RETiRet; +} + + +/* get the syslog forward port + * We may change the implementation to try to lookup the port + * if it is unspecified. So far, we use the IANA default auf 514. + * rgerhards, 2007-06-28 + */ +static inline uchar *getFwdPt(instanceData *pData) +{ + return (pData->port == NULL) ? UCHAR_CONSTANT("514") : pData->port; +} + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance + /* final cleanup */ + closeUDPSockets(pData); + free(pData->port); + free(pData->host); +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + DBGPRINTF("%s", pData->host); +ENDdbgPrintInstInfo + + +/* Send a message via UDP + * rgehards, 2007-12-20 + */ +static inline rsRetVal +UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) +{ + struct addrinfo *r; + int lsent = 0; + int bSendSuccess; + int j, build_ip; + u_char opt[20]; + struct sockaddr_in *tempaddr,source_ip; + libnet_ptag_t ip, ipo; + libnet_ptag_t udp; + DEFiRet; + + if(pData->pSockArray == NULL) { + CHKiRet(doTryResume(pData)); + } + + ip = ipo = udp = 0; + if(pData->sourcePort++ >= pData->sourcePortEnd){ + pData->sourcePort = pData->sourcePortStart; + } + + inet_pton(AF_INET, (char*)pszSourcename, &(source_ip.sin_addr)); + + bSendSuccess = FALSE; + for (r = pData->f_addr; r; r = r->ai_next) { + tempaddr = (struct sockaddr_in *)r->ai_addr; + libnet_clear_packet(libnet_handle); + /* note: libnet does need ports in host order NOT in network byte order! -- rgerhards, 2009-11-12 */ + udp = libnet_build_udp( + ntohs(pData->sourcePort),/* source port */ + ntohs(tempaddr->sin_port),/* destination port */ + LIBNET_UDP_H + len, /* packet length */ + 0, /* checksum */ + (u_char*)msg, /* payload */ + len, /* payload size */ + libnet_handle, /* libnet handle */ + udp); /* libnet id */ + if (udp == -1) { + DBGPRINTF("Can't build UDP header: %s\n", libnet_geterror(libnet_handle)); + } + + build_ip = 0; + /* this is not a legal options string */ + for (j = 0; j < 20; j++) { + opt[j] = libnet_get_prand(LIBNET_PR2); + } + ipo = libnet_build_ipv4_options(opt, 20, libnet_handle, ipo); + if (ipo == -1) { + DBGPRINTF("Can't build IP options: %s\n", libnet_geterror(libnet_handle)); + } + ip = libnet_build_ipv4( + LIBNET_IPV4_H + 20 + len + LIBNET_UDP_H, /* length */ + 0, /* TOS */ + 242, /* IP ID */ + 0, /* IP Frag */ + 64, /* TTL */ + IPPROTO_UDP, /* protocol */ + 0, /* checksum */ + source_ip.sin_addr.s_addr, + tempaddr->sin_addr.s_addr, + NULL, /* payload */ + 0, /* payload size */ + libnet_handle, /* libnet handle */ + ip); /* libnet id */ + if (ip == -1) { + DBGPRINTF("Can't build IP header: %s\n", libnet_geterror(libnet_handle)); + } + + /* Write it to the wire. */ + lsent = libnet_write(libnet_handle); + if (lsent == -1) { + DBGPRINTF("Write error: %s\n", libnet_geterror(libnet_handle)); + } else { + bSendSuccess = TRUE; + break; + } + } + /* finished looping */ + if (bSendSuccess == FALSE) { + DBGPRINTF("error forwarding via udp, suspending\n"); + iRet = RS_RET_SUSPENDED; + } + +finalize_it: + RETiRet; +} + + +/* try to resume connection if it is not ready + * rgerhards, 2007-08-02 + */ +static rsRetVal doTryResume(instanceData *pData) +{ + int iErr; + struct addrinfo *res; + struct addrinfo hints; + DEFiRet; + + if(pData->pSockArray != NULL) + FINALIZE; + + /* The remote address is not yet known and needs to be obtained */ + DBGPRINTF(" %s\n", pData->host); + memset(&hints, 0, sizeof(hints)); + /* port must be numeric, because config file syntax requires this */ + hints.ai_flags = AI_NUMERICSERV; + hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_socktype = SOCK_DGRAM; + if((iErr = (getaddrinfo((char*)pData->host, (char*)getFwdPt(pData), &hints, &res))) != 0) { + DBGPRINTF("could not get addrinfo for hostname '%s':'%s': %d%s\n", + pData->host, getFwdPt(pData), iErr, gai_strerror(iErr)); + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + DBGPRINTF("%s found, resuming.\n", pData->host); + pData->f_addr = res; + pData->pSockArray = net.create_udp_socket((uchar*)pData->host, NULL, 0); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pData->f_addr != NULL) { + freeaddrinfo(pData->f_addr); + pData->f_addr = NULL; + } + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +BEGINtryResume +CODESTARTtryResume + iRet = doTryResume(pData); +ENDtryResume + +BEGINdoAction + char *psz; /* temporary buffering */ + register unsigned l; + int iMaxLine; +CODESTARTdoAction + CHKiRet(doTryResume(pData)); + + iMaxLine = glbl.GetMaxLine(); + + DBGPRINTF(" %s:%s/udpspoofs\n", pData->host, getFwdPt(pData)); + + psz = (char*) ppString[0]; + l = strlen((char*) psz); + if((int) l > iMaxLine) + l = iMaxLine; + +# ifdef USE_NETZIP + /* Check if we should compress and, if so, do it. We also + * check if the message is large enough to justify compression. + * The smaller the message, the less likely is a gain in compression. + * To save CPU cycles, we do not try to compress very small messages. + * What "very small" means needs to be configured. Currently, it is + * hard-coded but this may be changed to a config parameter. + * rgerhards, 2006-11-30 + */ + if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { + Bytef *out; + uLongf destLen = iMaxLine + iMaxLine/100 +12; /* recommended value from zlib doc */ + uLong srcLen = l; + int ret; + /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ + CHKmalloc(out = (Bytef*) MALLOC(destLen)); + out[0] = 'z'; + out[1] = '\0'; + ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, + srcLen, pData->compressionLevel); + DBGPRINTF("Compressing message, length was %d now %d, return state %d.\n", + l, (int) destLen, ret); + if(ret != Z_OK) { + /* if we fail, we complain, but only in debug mode + * Otherwise, we are silent. In any case, we ignore the + * failed compression and just sent the uncompressed + * data, which is still valid. So this is probably the + * best course of action. + * rgerhards, 2006-11-30 + */ + DBGPRINTF("Compression failed, sending uncompressed message\n"); + } else if(destLen+1 < l) { + /* only use compression if there is a gain in using it! */ + DBGPRINTF("there is gain in compression, so we do it\n"); + psz = (char*) out; + l = destLen + 1; /* take care for the "z" at message start! */ + } + ++destLen; + } +# endif + + CHKiRet(UDPSend(pData, ppString[1], psz, l)); + +finalize_it: +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(2) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omudpspoof:", sizeof(":omudpspoof:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omudpspoof:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + if(pszSourceNameTemplate == NULL) { + errmsg.LogError(0, NO_ERRCODE, "No $ActionOMUDPSpoofSourceNameTemplate given, can not continue with this action."); + ABORT_FINALIZE(RS_RET_NO_SRCNAME_TPL); + } + + if(pszTargetHost == NULL) { + errmsg.LogError(0, NO_ERRCODE, "No $ActionOMUDPSpoofTargetHost given, can not continue with this action."); + ABORT_FINALIZE(RS_RET_HOST_NOT_SPECIFIED); + } + + /* fill instance properties */ + CHKmalloc(pData->host = ustrdup(pszTargetHost)); + if(pszTargetPort == NULL) + pData->port = NULL; + else + CHKmalloc(pData->port = ustrdup(pszTargetPort)); + CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(pszSourceNameTemplate), OMSR_NO_RQD_TPL_OPTS)); + pData->compressionLevel = iCompressionLevel; + pData->sourcePort = pData->sourcePortStart = iSourcePortStart; + pData->sourcePortEnd = iSourcePortEnd; + + /* process template */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_TraditionalForwardFormat" : pszTplName)); + +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +/* a common function to free our configuration variables - used both on exit + * and on $ResetConfig processing. -- rgerhards, 2008-05-16 + */ +static void +freeConfigVars(void) +{ + free(pszTplName); + pszTplName = NULL; + free(pszTargetHost); + pszTargetHost = NULL; + free(pszTargetPort); + pszTargetPort = NULL; +} + + +BEGINmodExit +CODESTARTmodExit + /* destroy the libnet state needed for forged UDP sources */ + libnet_destroy(libnet_handle); + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(net, LM_NET_FILENAME); + freeConfigVars(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +/* Reset config variables for this module to default values. + * rgerhards, 2008-03-28 + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + freeConfigVars(); + /* we now must reset all non-string values */ + iCompressionLevel = 0; + iSourcePortStart = DFLT_SOURCE_PORT_START; + iSourcePortEnd = DFLT_SOURCE_PORT_END; + return RS_RET_OK; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(net,LM_NET_FILENAME)); + + /* Initialize the libnet library. Root priviledges are required. + * this initializes a IPv4 socket to use for forging UDP packets. + */ + libnet_handle = libnet_init( + LIBNET_RAW4, /* injection type */ + NULL, /* network interface */ + errbuf); /* errbuf */ + + if(libnet_handle == NULL) { + errmsg.LogError(0, NO_ERRCODE, "Error initializing libnet, can not continue "); + ABORT_FINALIZE(RS_RET_ERR_LIBNET_INIT); + } + + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofdefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszTplName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofsourcenametemplate", 0, eCmdHdlrGetWord, NULL, &pszSourceNameTemplate, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspooftargethost", 0, eCmdHdlrGetWord, NULL, &pszTargetHost, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspooftargetport", 0, eCmdHdlrGetWord, NULL, &pszTargetPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofsourceportstart", 0, eCmdHdlrInt, NULL, &iSourcePortStart, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofsourceportend", 0, eCmdHdlrInt, NULL, &iSourcePortEnd, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpcompressionlevel", 0, eCmdHdlrInt, NULL, &iCompressionLevel, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vim:set ai: + */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index bc03c4a7..9047c83d 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -9,12 +9,14 @@ librsyslog_la_SOURCES = \ rsyslog.h \ unicode-helper.h \ atomic.h \ + batch.h \ syslogd-types.h \ module-template.h \ obj-types.h \ nsd.h \ glbl.h \ glbl.c \ + unlimited_select.h \ conf.c \ conf.h \ parser.h \ @@ -39,6 +41,8 @@ librsyslog_la_SOURCES = \ obj.h \ modules.c \ modules.h \ + apc.c \ + apc.h \ sync.c \ sync.h \ expr.c \ @@ -67,6 +71,12 @@ librsyslog_la_SOURCES = \ vmop.h \ queue.c \ queue.h \ + ruleset.c \ + ruleset.h \ + rule.c \ + rule.h \ + prop.c \ + prop.h \ cfsysline.c \ cfsysline.h \ \ @@ -105,6 +115,17 @@ lmregexp_la_LDFLAGS = -module -avoid-version lmregexp_la_LIBADD = endif +# +# zlib support +# +if ENABLE_ZLIB +pkglib_LTLIBRARIES += lmzlibw.la +lmzlibw_la_SOURCES = zlibw.c zlibw.h +lmzlibw_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmzlibw_la_LDFLAGS = -module -avoid-version +lmzlibw_la_LIBADD = +endif + if ENABLE_INET pkglib_LTLIBRARIES += lmnet.la lmnetstrms.la # @@ -116,16 +137,28 @@ lmnet_la_LDFLAGS = -module -avoid-version lmnet_la_LIBADD = # network stream master class and stream factory -lmnetstrms_la_SOURCES = netstrms.c netstrms.h netstrm.c netstrm.h nssel.c nssel.h +lmnetstrms_la_SOURCES = netstrms.c netstrms.h \ + netstrm.c netstrm.h \ + nssel.c nssel.h \ + nspoll.c nspoll.h lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version lmnetstrms_la_LIBADD = +# generic stream server framework +pkglib_LTLIBRARIES += lmstrmsrv.la +lmstrmsrv_la_SOURCES = strmsrv.c strmsrv.h strms_sess.c strms_sess.h +lmstrmsrv_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmstrmsrv_la_LDFLAGS = -module -avoid-version +lmstrmsrv_la_LIBADD = + # netstream drivers # plain tcp driver - main driver pkglib_LTLIBRARIES += lmnsd_ptcp.la -lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h nsdsel_ptcp.c nsdsel_ptcp.h +lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h \ + nsdsel_ptcp.c nsdsel_ptcp.h \ + nsdpoll_ptcp.c nsdpoll_ptcp.h lmnsd_ptcp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnsd_ptcp_la_LDFLAGS = -module -avoid-version lmnsd_ptcp_la_LIBADD = diff --git a/runtime/apc.c b/runtime/apc.c new file mode 100644 index 00000000..3c6b7ec4 --- /dev/null +++ b/runtime/apc.c @@ -0,0 +1,402 @@ +/* apc.c - asynchronous procedure call support + * + * An asynchronous procedure call (APC) is a procedure call (guess what) that is potentially run + * asynchronously to its main thread. It can be scheduled to occur at a caller-provided time. + * As long as the procedure has not been called, the APC entry may be modified by the caller + * or deleted. It is the caller's purpose to make sure proper synchronization is in place. + * The APC object only case about APC's own control structures (which *are* properly + * guarded by synchronization primitives). + * + * Module begun 2009-06-15 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <pthread.h> + +#include "rsyslog.h" +#include "obj.h" +#include "apc.h" +#include "srUtils.h" +#include "datetime.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(datetime) + +/* following is a used to implement a monotonically increasing id for the apcs. That + * ID can be used to cancel an apc request. Note that the ID is generated with modulo + * arithmetic, so at some point, it will wrap. Howerver, this happens at 2^32-1 at + * earliest, so this is not considered a problem. + */ +apc_id_t apcID = 0; + +/* private data structures */ + +/* the apc list and its entries + * This is a doubly-linked list as we need to be able to do inserts + * and deletes right in the middle of the list. It is inspired by the + * Unix callout mechanism. + * Note that we support two generic caller-provided parameters as + * experience shows that at most two are often used. This causes very + * little overhead, but simplifies caller code in cases where exactly + * two parameters are needed. We hope this is a useful optimizaton. + * rgerhards, 2009-06-15 + */ +typedef struct apc_list_s { + struct apc_list_s *pNext; + struct apc_list_s *pPrev; + apc_id_t id; + apc_t *pApc; /* pointer to the APC object to be scheduled */ +} apc_list_t; + +apc_list_t *apcListRoot = NULL; +apc_list_t *apcListTail = NULL; +pthread_mutex_t listMutex; /* needs to be locked for all list operations */ + + +/* destructor for the apc object */ +BEGINobjDestruct(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(apc) +ENDobjDestruct(apc) + + +/* ------------------------------ APC list handling functions ------------------------------ */ + +/* Function that handles changes to the list root. Most importantly, this function + * needs to schedule a new timer. It is OK to call this function with an empty list. + */ +static rsRetVal +listRootChanged(void) +{ + DEFiRet; + + if(apcListRoot == NULL) + FINALIZE; + + // TODO: implement! + +finalize_it: + RETiRet; +} + + +/* insert an apc entry into the APC list. The same entry MUST NOT already be present! + */ +static rsRetVal +insertApc(apc_t *pThis, apc_id_t *pID) +{ + apc_list_t *pCurr; + apc_list_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (apc_list_t*) calloc(1, sizeof(apc_list_t))); + pNew->pApc = pThis; + pNew->id = *pID = apcID++; +dbgprintf("insert apc %p, id %ld\n", pThis, pNew->id); + + /* find right list location */ + if(apcListRoot == NULL) { + /* no need to search, list is empty */ + apcListRoot = pNew; + apcListTail = pNew; + CHKiRet(listRootChanged()); + } else { + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->pApc->ttExec > pThis->ttExec) + break; + } + + if(pCurr == NULL) { + /* insert at tail */ + pNew->pPrev = apcListTail; + apcListTail->pNext = pNew; + apcListTail = pNew; + } else { + if(pCurr == apcListRoot) { + /* new first entry */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + apcListRoot = pNew; + CHKiRet(listRootChanged()); + } else { + /* in the middle of the list */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + } + } + } + + +finalize_it: + RETiRet; +} + + +/* Delete an apc entry from the APC list. It is OK if the entry is not found, + * in this case we assume it already has been processed. + */ +static rsRetVal +deleteApc(apc_id_t id) +{ + apc_list_t *pCurr; + DEFiRet; + +dbgprintf("trying to delete apc %ld\n", id); + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->id == id) { +RUNLOG_STR("apc id found, now deleting!\n"); + if(pCurr == apcListRoot) { + apcListRoot = pCurr->pNext; + CHKiRet(listRootChanged()); + } else { + pCurr->pPrev->pNext = pCurr->pNext; + } + if(pCurr->pNext == NULL) { + apcListTail = pCurr->pPrev; + } else { + pCurr->pNext->pPrev = pCurr->pPrev; + } + free(pCurr); + pCurr = NULL; + break; + } + } + +finalize_it: + RETiRet; +} + + +/* unlist all elements up to the current timestamp. Return this as a seperate list + * to the caller. Returns an empty (NULL ptr) list if there are no such elements. + * The caller must handle that gracefully. The list is returned in the parameter. + */ +static rsRetVal +unlistCurrent(apc_list_t **ppList) +{ + apc_list_t *pCurr; + time_t tCurr; + DEFiRet; + assert(ppList != NULL); + + datetime.GetTime(&tCurr); + + if(apcListRoot == NULL || apcListRoot->pApc->ttExec > tCurr) { + *ppList = NULL; + FINALIZE; + } + + *ppList = apcListRoot; + /* now search up to which entry we need to execute */ + for(pCurr = apcListRoot ; pCurr != NULL && pCurr->pApc->ttExec <= tCurr ; pCurr = pCurr->pNext) { + /*JUST SKIP TO LAST ELEMENT*/; + } + + if(pCurr == NULL) { + /* all elements can be unlisted */ + apcListRoot = NULL; + apcListTail = NULL; + } else { + /* need to set a new root */ + pCurr->pPrev->pNext = NULL; /* terminate newly unlisted list */ + pCurr->pPrev = NULL; /* we are the new root */ + apcListRoot = pCurr; + } + +finalize_it: + RETiRet; +} + + +/* ------------------------------ END APC list handling functions ------------------------------ */ + + +/* execute all list elements that are currently scheduled for execution. We do this in two phases. + * In the first phase, we look the list mutex and move everything from the head of the queue to + * the current timestamp to a new to-be-executed list. Then we unlock the mutex and do the actual + * exec (which may take some time). + * Note that the caller is responsible for proper + * caller-level synchronization. The caller may schedule another Apc, this module must + * ensure that (and it does so by not locking the list mutex while we call the Apc). + * Note: this function "consumes" the apc_t, so it is no longer existing after this + * function returns. + */ +// TODO make static and associated with our own pthread-based timer +rsRetVal +execScheduled(void) +{ + apc_list_t *pExecList; + apc_list_t *pCurr; + apc_list_t *pNext; + DEFiRet; + + d_pthread_mutex_lock(&listMutex); + iRet = unlistCurrent(&pExecList); + d_pthread_mutex_unlock(&listMutex); + CHKiRet(iRet); + + if(pExecList != NULL) { + DBGPRINTF("running apc scheduler - we have %s to execute\n", + pExecList == NULL ? "nothing" : "something"); + } + + for(pCurr = pExecList ; pCurr != NULL ; pCurr = pNext) { +dbgprintf("executing apc list entry %p, apc %p\n", pCurr, pCurr->pApc); + pNext = pCurr->pNext; + pCurr->pApc->pProc(pCurr->pApc->param1, pCurr->pApc->param2); + apcDestruct(&pCurr->pApc); + free(pCurr); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(apc) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(apc) + + +/* ConstructionFinalizer + * Note that we use a non-standard calling interface: pID returns the current APC + * id. This is the only way to handle the situation without the need for extra + * locking. + * rgerhards, 2008-01-09 + */ +static rsRetVal +apcConstructFinalize(apc_t *pThis, apc_id_t *pID) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, apc); + assert(pID != NULL); + d_pthread_mutex_lock(&listMutex); + insertApc(pThis, pID); + d_pthread_mutex_unlock(&listMutex); + RETiRet; +} + + +/* some set methods */ +static rsRetVal +SetProcedure(apc_t *pThis, void (*pProc)(void*, void*)) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->pProc = pProc; + return RS_RET_OK; +} +static rsRetVal +SetParam1(apc_t *pThis, void *param1) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param1; + return RS_RET_OK; +} +static rsRetVal +SetParam2(apc_t *pThis, void *param2) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param2; + return RS_RET_OK; +} + + +/* cancel an Apc request, ID is provided. It is OK if the ID can not be found, this may + * happen if the Apc was executed in the mean time. So it is safe to call CancelApc() at + * any time. + */ +static rsRetVal +CancelApc(apc_id_t id) +{ + BEGINfunc + d_pthread_mutex_lock(&listMutex); + deleteApc(id); + d_pthread_mutex_unlock(&listMutex); + ENDfunc + return RS_RET_OK; +} + + +/* debugprint for the apc object */ +BEGINobjDebugPrint(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(apc) + dbgoprint((obj_t*) pThis, "APC module, currently no state info available\n"); +ENDobjDebugPrint(apc) + + +/* queryInterface function + */ +BEGINobjQueryInterface(apc) +CODESTARTobjQueryInterface(apc) + if(pIf->ifVersion != apcCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = apcConstruct; + pIf->ConstructFinalize = apcConstructFinalize; + pIf->Destruct = apcDestruct; + pIf->DebugPrint = apcDebugPrint; + pIf->CancelApc = CancelApc; + pIf->SetProcedure = SetProcedure; + pIf->SetParam1 = SetParam1; + pIf->SetParam2 = SetParam2; +finalize_it: +ENDobjQueryInterface(apc) + + +/* Exit the apc class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(apc, OBJ_IS_CORE_MODULE) /* class, version */ + objRelease(datetime, CORE_COMPONENT); + pthread_mutex_destroy(&listMutex); +ENDObjClassExit(apc) + + +/* Initialize the apc class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(apc, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, apcDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, apcConstructFinalize); + + /* do other initializations */ + pthread_mutex_init(&listMutex, NULL); +ENDObjClassInit(apc) + +/* vi:set ai: + */ diff --git a/runtime/apc.h b/runtime/apc.h new file mode 100644 index 00000000..7c679b97 --- /dev/null +++ b/runtime/apc.h @@ -0,0 +1,56 @@ +/* The apc object. + * + * See apc.c for more information. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_APC_H +#define INCLUDED_APC_H + +/* the apc object */ +typedef struct apc_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + time_t ttExec; /* when to call procedure (so far seconds...) */ + void (*pProc)(void*, void*); /* which procedure to call */ + void *param1; /* user-supplied parameters */ + void *param2; /* user-supplied parameters */ +} apc_t; + +typedef unsigned long apc_id_t; /* monotonically incrementing apc ID */ + +/* interfaces */ +BEGINinterface(apc) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(apc); + rsRetVal (*Construct)(apc_t **ppThis); + rsRetVal (*ConstructFinalize)(apc_t *pThis, apc_id_t *); + rsRetVal (*Destruct)(apc_t **ppThis); + rsRetVal (*SetProcedure)(apc_t *pThis, void (*pProc)(void*, void*)); + rsRetVal (*SetParam1)(apc_t *pThis, void *); + rsRetVal (*SetParam2)(apc_t *pThis, void *); + rsRetVal (*CancelApc)(apc_id_t); +ENDinterface(apc) +#define apcCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(apc); + +#endif /* #ifndef INCLUDED_APC_H */ diff --git a/runtime/atomic.h b/runtime/atomic.h index fdf64214..b507b769 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -41,11 +41,18 @@ * They simply came in too late. -- rgerhards, 2008-04-02 */ #ifdef HAVE_ATOMIC_BUILTINS +# define ATOMIC_SUB(data, val) __sync_fetch_and_sub(&(data), val) +# define ATOMIC_ADD(data, val) __sync_fetch_and_add(&(data), val) # define ATOMIC_INC(data) ((void) __sync_fetch_and_add(&(data), 1)) +# define ATOMIC_INC_AND_FETCH(data) __sync_fetch_and_add(&(data), 1) # define ATOMIC_DEC(data) ((void) __sync_sub_and_fetch(&(data), 1)) # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) +# define ATOMIC_STORE_0_TO_INT(data) __sync_fetch_and_and(&(data), 0) +# define ATOMIC_STORE_1_TO_INT(data) __sync_fetch_and_or(&(data), 1) +# define ATOMIC_CAS(data, oldVal, newVal) __sync_bool_compare_and_swap(&(data), (oldVal), (newVal)); +# define ATOMIC_CAS_VAL(data, oldVal, newVal) __sync_val_compare_and_swap(&(data), (oldVal), (newVal)); #else /* note that we gained parctical proof that theoretical problems DO occur * if we do not properly address them. See this blog post for details: diff --git a/runtime/batch.h b/runtime/batch.h new file mode 100644 index 00000000..2b3aa83e --- /dev/null +++ b/runtime/batch.h @@ -0,0 +1,72 @@ +/* Definition of the batch_t data structure. + * I am not sure yet if this will become a full-blown object. For now, this header just + * includes the object definition and is not accompanied by code. + * + * Copyright 2009 by Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef BATCH_H_INCLUDED +#define BATCH_H_INCLUDED + +/* enum for batch states. Actually, we violate a layer here, in that we assume that a batch is used + * for action processing. So far, this seems acceptable, the status is simply ignored inside the + * main message queue. But over time, it could potentially be useful to split the two. + * rgerhad, 2009-05-12 + */ +typedef enum { + BATCH_STATE_RDY = 0, /* object ready for processing */ + BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ + BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unknown */ + BATCH_STATE_COMM = 3, /* message successfully commited */ + BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */ +} batch_state_t; + + +/* an object inside a batch, including any information (state!) needed for it to "life". + */ +struct batch_obj_s { + obj_t *pUsrp; /* pointer to user object (most often message) */ + batch_state_t state; /* associated state */ +}; + +/* the batch + * This object is used to dequeue multiple user pointers which are than handed over + * to processing. The size of elements is fixed after queue creation, but may be + * modified by config variables (better said: queue properties). + * Note that a "user pointer" in rsyslog context so far always is a message + * object. We stick to the more generic term because queues may potentially hold + * other types of objects, too. + * rgerhards, 2009-05-12 + * Note that nElem is not necessarily equal to nElemDeq. This is the case when we + * discard some elements (because of configuration) during dequeue processing. As + * all Elements are only deleted when the batch is processed, we can not immediately + * delete them. So we need to keep their number that we can delete them when the batch + * is completed (else, the whole process does not work correctly). + */ +struct batch_s { + int nElem; /* actual number of element in this entry */ + int nElemDeq; /* actual number of elements dequeued (and thus to be deleted) - see comment above! */ + int iDoneUpTo; /* all messages below this index have state other than RDY */ + qDeqID deqID; /* ID of dequeue operation that generated this batch */ + batch_obj_t *pElem; /* batch elements */ +}; + +#endif /* #ifndef BATCH_H_INCLUDED */ diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index e1e4a6a4..184c0d87 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -462,7 +462,7 @@ getWord(uchar **pp, cstr_t **ppStrB) ASSERT(*pp != NULL); ASSERT(ppStrB != NULL); - CHKiRet(rsCStrConstruct(ppStrB)); + CHKiRet(cstrConstruct(ppStrB)); skipWhiteSpace(pp); /* skip over any whitespace */ @@ -470,9 +470,9 @@ getWord(uchar **pp, cstr_t **ppStrB) p = *pp; while(*p && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(*ppStrB, *p++)); + CHKiRet(cstrAppendChar(*ppStrB, *p++)); } - CHKiRet(rsCStrFinish(*ppStrB)); + CHKiRet(cstrFinalize(*ppStrB)); *pp = p; @@ -506,7 +506,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); pStrB = NULL; /* we got the word, now set it */ @@ -525,7 +525,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -548,7 +548,7 @@ doSyslogName(uchar **pp, rsRetVal (*pSetHdlr)(void*, int), void *pVal, syslogNam ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); /* get word */ - iNewVal = decodeSyslogName(rsCStrGetSzStr(pStrB), pNameTable); + iNewVal = decodeSyslogName(cstrGetSzStr(pStrB), pNameTable); if(pSetHdlr == NULL) { /* we should set value directly to var */ @@ -814,7 +814,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy CHKiRet(cslcConstruct(&pThis, bChainingPermitted)); CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } /* important: add to list, AFTER everything else is OK. Else * we mess up things in the error case. @@ -825,7 +825,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(llAppend(&llCmdList, pMyCmdName, (void*) pThis)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } else { /* command already exists, are we allowed to chain? */ @@ -834,7 +834,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } diff --git a/runtime/conf.c b/runtime/conf.c index 27ab8bb4..96d9dcab 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -69,13 +69,16 @@ #include "expr.h" #include "ctok.h" #include "ctok_token.h" +#include "rule.h" +#include "ruleset.h" +#include "unicode-helper.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN #endif /* forward definitions */ -static rsRetVal cfline(uchar *line, selector_t **pfCurr); +static rsRetVal cfline(uchar *line, rule_t **pfCurr); static rsRetVal processConfFile(uchar *pConfFile); @@ -87,20 +90,21 @@ DEFobjCurrIf(ctok_token) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) +DEFobjCurrIf(rule) +DEFobjCurrIf(ruleset) -static int iNbrActions; /* number of actions the running config has. Needs to be init on ReInitConf() */ +static int iNbrActions = 0; /* number of currently defined actions */ -/* The following global variables are used for building +/* The following module-global variables are used for building * tag and host selector lines during startup and config reload. * This is stored as a global variable pool because of its ease. It is * also fairly compatible with multi-threading as the stratup code must - * be run in a single thread anyways. So there can be no race conditions. These - * variables are no longer used once the configuration has been loaded (except, - * of course, during a reload). rgerhards 2005-10-18 + * be run in a single thread anyways. So there can be no race conditions. + * rgerhards 2005-10-18 */ -EHostnameCmpMode eDfltHostnameCmpMode; -cstr_t *pDfltHostnameCmp; -cstr_t *pDfltProgNameCmp; +static EHostnameCmpMode eDfltHostnameCmpMode = HN_NO_COMP; +static cstr_t *pDfltHostnameCmp = NULL; +static cstr_t *pDfltProgNameCmp = NULL; /* process a directory and include all of its files into @@ -211,7 +215,7 @@ doIncludeLine(uchar **pp, __attribute__((unused)) void* pVal) * Required by doIncludeDirectory(). */ result = glob(pattern, GLOB_MARK, NULL, &cfgFiles); - if(result != 0) { + if(result == GLOB_NOSPACE || result == GLOB_ABORTED) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); errmsg.LogError(0, RS_RET_FILE_NOT_FOUND, "error accessing config file or directory '%s': %s", @@ -392,14 +396,17 @@ finalize_it: static rsRetVal processConfFile(uchar *pConfFile) { - DEFiRet; int iLnNbr = 0; FILE *cf; - selector_t *fCurr = NULL; + rule_t *pCurrRule = NULL; uchar *p; uchar cbuf[CFGLNSIZ]; uchar *cline; int i; + int bHadAnError = 0; + uchar *pszOrgLine = NULL; + size_t lenLine; + DEFiRet; ASSERT(pConfFile != NULL); if((cf = fopen((char*)pConfFile, "r")) == NULL) { @@ -412,9 +419,12 @@ processConfFile(uchar *pConfFile) while (fgets((char*)cline, sizeof(cbuf) - (cline - cbuf), cf) != NULL) { ++iLnNbr; /* drop LF - TODO: make it better, replace fgets(), but its clean as it is */ - if(cline[strlen((char*)cline)-1] == '\n') { - cline[strlen((char*)cline) -1] = '\0'; + lenLine = ustrlen(cline); + if(cline[lenLine-1] == '\n') { + cline[lenLine-1] = '\0'; } + free(pszOrgLine); + pszOrgLine = ustrdup(cline); /* save if needed for errmsg, NULL ptr is OK */ /* check for end-of-section, comments, strip off trailing * spaces and newline character. */ @@ -428,7 +438,6 @@ processConfFile(uchar *pConfFile) * TODO: review the code at whole - this is highly suspect (but will go away * once we do the rest of RainerScript). */ - /* was: strcpy((char*)cline, (char*)p); */ for( i = 0 ; p[i] != '\0' ; ++i) { cline[i] = p[i]; } @@ -452,7 +461,7 @@ processConfFile(uchar *pConfFile) /* we now have the complete line, and are positioned at the first non-whitespace * character. So let's process it */ - if(cfline(cbuf, &fCurr) != RS_RET_OK) { + if(cfline(cbuf, &pCurrRule) != RS_RET_OK) { /* we log a message, but otherwise ignore the error. After all, the next * line can be correct. -- rgerhards, 2007-08-02 */ @@ -460,26 +469,35 @@ processConfFile(uchar *pConfFile) dbgprintf("config line NOT successfully processed\n"); snprintf((char*)szErrLoc, sizeof(szErrLoc) / sizeof(uchar), "%s, line %d", pConfFile, iLnNbr); - errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s", (char*)szErrLoc); + errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s:\"%s\"", (char*)szErrLoc, (char*)pszOrgLine); + bHadAnError = 1; } } /* we probably have one selector left to be added - so let's do that now */ - CHKiRet(selectorAddList(fCurr)); + if(pCurrRule != NULL) { + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(pCurrRule), &pCurrRule)); + } /* close the configuration file */ - (void) fclose(cf); + fclose(cf); finalize_it: if(iRet != RS_RET_OK) { char errStr[1024]; - if(fCurr != NULL) - selectorDestruct(fCurr); + if(pCurrRule != NULL) + rule.Destruct(&pCurrRule); rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("error %d processing config file '%s'; os error (if any): %s\n", iRet, pConfFile, errStr); } + + free(pszOrgLine); + + if(bHadAnError && (iRet == RS_RET_OK)) { /* a bit dirty, enhance in future releases */ + iRet = RS_RET_NONFATAL_CONFIG_ERR; + } RETiRet; } @@ -494,7 +512,7 @@ finalize_it: rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName) { uchar *p; - uchar *tplName; + uchar *tplName = NULL; cstr_t *pStrB; DEFiRet; @@ -520,23 +538,23 @@ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEn tplName = (uchar*) strdup((char*)dfltTplName); } else { /* template specified, pick it up */ - if(rsCStrConstruct(&pStrB) != RS_RET_OK) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKiRet(cstrConstruct(&pStrB)); /* now copy the string */ while(*p && *p != '#' && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &tplName, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &tplName, 0)); } - iRet = OMSRsetEntry(pOMSR, iEntry, tplName, iTplOpts); - if(iRet != RS_RET_OK) goto finalize_it; + CHKiRet(OMSRsetEntry(pOMSR, iEntry, tplName, iTplOpts)); finalize_it: + if(iRet != RS_RET_OK) + free(tplName); + *pp = p; RETiRet; @@ -551,6 +569,7 @@ finalize_it: * rgerhards, 2007-07-25 * updated to include OMSR pointer -- rgerhards, 2007-07-27 * updated to include template name -- rgerhards, 2008-03-28 + * rgerhards, 2010-01-19: file names end at the first space */ rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl) @@ -563,7 +582,7 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int pName = pFileName; i = 1; /* we start at 1 so that we reseve space for the '\0'! */ - while(*p && *p != ';' && i < MAXFNAME) { + while(*p && *p != ';' && *p != ' ' && i < MAXFNAME) { *pName++ = *p++; ++i; } @@ -582,7 +601,7 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int * rgerhards 2005-09-15 */ /* GPLv3 - stems back to sysklogd */ -static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) { uchar *p; register uchar *q; @@ -597,17 +616,17 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f ASSERT(pline != NULL); ASSERT(*pline != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(pRule, rule); dbgprintf(" - traditional PRI filter\n"); errno = 0; /* keep strerror_r() stuff out of logerror messages */ - f->f_filter_type = FILTER_PRI; + pRule->f_filter_type = FILTER_PRI; /* Note: file structure is pre-initialized to zero because it was * created with calloc()! */ for (i = 0; i <= LOG_NFACILITIES; i++) { - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } /* scan through the list of selectors */ @@ -662,32 +681,32 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f for (i = 0; i <= LOG_NFACILITIES; i++) { if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] &= ~(1<<pri); + pRule->f_filterData.f_pmask[i] &= ~(1<<pri); else - f->f_filterData.f_pmask[i] |= (1<<pri); + pRule->f_filterData.f_pmask[i] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i] &= ~(1<<i2); + pRule->f_filterData.f_pmask[i] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i] |= (1<<i2); + pRule->f_filterData.f_pmask[i] |= (1<<i2); } } } @@ -702,27 +721,27 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] &= ~(1<<pri); + pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<pri); else - f->f_filterData.f_pmask[i >> 3] |= (1<<pri); + pRule->f_filterData.f_pmask[i >> 3] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i >> 3] &= ~(1<<i2); + pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i >> 3] |= (1<<i2); + pRule->f_filterData.f_pmask[i >> 3] |= (1<<i2); } } } @@ -748,7 +767,7 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f * A pointer to that beginnig is passed back to the caller. * rgerhards 2008-01-19 */ -static rsRetVal cflineProcessIfFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessIfFilter(uchar **pline, register rule_t *f) { DEFiRet; ctok_t *tok; @@ -761,7 +780,6 @@ static rsRetVal cflineProcessIfFilter(uchar **pline, register selector_t *f) dbgprintf(" - general expression-based filter\n"); errno = 0; /* keep strerror_r() stuff out of logerror messages */ -dbgprintf("calling expression parser, pp %p ('%s')\n", *pline, *pline); f->f_filter_type = FILTER_EXPR; /* if we come to over here, pline starts with "if ". We just skip that part. */ @@ -817,10 +835,11 @@ finalize_it: * of the action part. A pointer to that beginnig is passed back to the caller. * rgerhards 2005-09-15 */ -static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) { rsParsObj *pPars; cstr_t *pCSCompOp; + cstr_t *pCSPropName; rsRetVal iRet; int iOffset; /* for compare operations */ @@ -840,12 +859,19 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) } /* read property */ - iRet = parsDelimCStr(pPars, &f->f_filterData.prop.pCSPropName, ',', 1, 1, 1); + iRet = parsDelimCStr(pPars, &pCSPropName, ',', 1, 1, 1); + if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); + rsParsDestruct(pPars); + return(iRet); + } + iRet = propNameToID(pCSPropName, &f->f_filterData.prop.propID); if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); rsParsDestruct(pPars); return(iRet); } + cstrDestruct(&pCSPropName); /* read operation */ iRet = parsDelimCStr(pPars, &pCSCompOp, ',', 1, 1, 1); @@ -919,7 +945,7 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) */ static rsRetVal cflineProcessHostSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -945,21 +971,20 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) dbgprintf("resetting BSD-like hostname filter\n"); eDfltHostnameCmpMode = HN_NO_COMP; if(pDfltHostnameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, NULL)); } } else { dbgprintf("setting BSD-like hostname filter to '%s'\n", *pline); if(pDfltHostnameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } @@ -970,7 +995,7 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) */ static rsRetVal cflineProcessTagSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -989,34 +1014,33 @@ static rsRetVal cflineProcessTagSelector(uchar **pline) if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { dbgprintf("resetting programname filter\n"); if(pDfltProgNameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, NULL)); } } else { dbgprintf("setting programname filter to '%s'\n", *pline); if(pDfltProgNameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } /* read the filter part of a configuration line and store the filter - * in the supplied selector_t + * in the supplied rule_t * rgerhards, 2007-08-01 */ -static rsRetVal cflineDoFilter(uchar **pp, selector_t *f) +static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) { DEFiRet; ASSERT(pp != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(f, rule); /* check which filter we need to pull... */ switch(**pp) { @@ -1038,6 +1062,7 @@ static rsRetVal cflineDoFilter(uchar **pp, selector_t *f) * and, if so, we copy them over. rgerhards, 2005-10-18 */ if(pDfltProgNameCmp != NULL) { +RUNLOG_STR("dflt ProgNameCmp != NULL, setting opCSProgNameComp"); CHKiRet(rsCStrConstructFromCStr(&(f->pCSProgNameComp), pDfltProgNameCmp)); } @@ -1080,7 +1105,7 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); pAction->f_ReduceRepeated = 0; } - pAction->bEnabled = 1; /* action is enabled */ + pAction->eState = ACT_STATE_RDY; /* action is enabled */ iNbrActions++; /* one more active action! */ } break; @@ -1104,17 +1129,15 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) /* Process a configuration file line in traditional "filter selector" format - * or one that builds upon this format. + * or one that builds upon this format. Note that ppRule may be a NULL pointer, + * which is valid and happens if there is no previous line (right at the start + * of the master config file!). */ -static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) +static rsRetVal +cflineClassic(uchar *p, rule_t **ppRule) { DEFiRet; action_t *pAction; - selector_t *fCurr; - - ASSERT(pfCurr != NULL); - - fCurr = *pfCurr; /* lines starting with '&' have no new filters and just add * new actions to the currently processed selector. @@ -1132,16 +1155,19 @@ static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) * selector is NULL, which means we do not need to care about it at * all. -- rgerhards, 2007-08-01 */ - CHKiRet(selectorAddList(fCurr)); - CHKiRet(selectorConstruct(&fCurr)); /* create "fresh" selector */ - CHKiRet(cflineDoFilter(&p, fCurr)); /* pull filters */ + if(*ppRule != NULL) { + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(*ppRule), ppRule)); + } + CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ + CHKiRet(rule.SetAssRuleset(*ppRule, ruleset.GetCurrent())); /* create "fresh" selector */ + CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ + CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } CHKiRet(cflineDoAction(&p, &pAction)); - CHKiRet(llAppend(&fCurr->llActList, NULL, (void*) pAction)); + CHKiRet(llAppend(&(*ppRule)->llActList, NULL, (void*) pAction)); finalize_it: - *pfCurr = fCurr; RETiRet; } @@ -1151,7 +1177,7 @@ finalize_it: * rgerhards, 2007-08-01 */ static rsRetVal -cfline(uchar *line, selector_t **pfCurr) +cfline(uchar *line, rule_t **pfCurr) { DEFiRet; @@ -1181,21 +1207,6 @@ cfline(uchar *line, selector_t **pfCurr) } -/* Reinitialize the configuration subsystem. This is a "work-around" to the fact - * that we do not yet have actual config objects. This method is to be called - * whenever a totally new config is started (which means on startup and HUP). - * Note that it MUST NOT be called for an included config file. - * rgerhards, 2008-07-28 - */ -static rsRetVal -ReInitConf(void) -{ - DEFiRet; - iNbrActions = 0; /* this is what we created the function for ;) - action count is reset */ - RETiRet; -} - - /* return the current number of active actions * rgerhards, 2008-07-28 */ @@ -1229,7 +1240,6 @@ CODESTARTobjQueryInterface(conf) pIf->doIncludeLine = doIncludeLine; pIf->cfline = cfline; pIf->processConfFile = processConfFile; - pIf->ReInitConf = ReInitConf; pIf->GetNbrActActions = GetNbrActActions; finalize_it: @@ -1241,6 +1251,15 @@ ENDobjQueryInterface(conf) */ BEGINObjClassExit(conf, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ CODESTARTObjClassExit(conf) + /* free no-longer needed module-global variables */ + if(pDfltHostnameCmp != NULL) { + rsCStrDestruct(&pDfltHostnameCmp); + } + + if(pDfltProgNameCmp != NULL) { + rsCStrDestruct(&pDfltProgNameCmp); + } + /* release objects we no longer need */ objRelease(expr, CORE_COMPONENT); objRelease(ctok, CORE_COMPONENT); @@ -1248,6 +1267,8 @@ CODESTARTObjClassExit(conf) objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); + objRelease(rule, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDObjClassExit(conf) @@ -1263,6 +1284,8 @@ BEGINAbstractObjClassInit(conf, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANG CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* TODO: make this dependcy go away! */ + CHKiRet(objUse(rule, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); ENDObjClassInit(conf) /* vi:set ai: diff --git a/runtime/conf.h b/runtime/conf.h index 2494d4dc..d85d1f82 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -35,21 +35,23 @@ BEGINinterface(conf) /* name must also be changed in ENDinterface macro! */ rsRetVal (*cfsysline)(uchar *p); rsRetVal (*doModLoad)(uchar **pp, __attribute__((unused)) void* pVal); rsRetVal (*doIncludeLine)(uchar **pp, __attribute__((unused)) void* pVal); - rsRetVal (*cfline)(uchar *line, selector_t **pfCurr); + rsRetVal (*cfline)(uchar *line, rule_t **pfCurr); rsRetVal (*processConfFile)(uchar *pConfFile); - rsRetVal (*ReInitConf)(void); rsRetVal (*GetNbrActActions)(int *); ENDinterface(conf) -#define confCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define confCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ +/* in Version 3, entry point "ReInitConf()" was removed, as we do not longer need + * to support restart-type HUP -- rgerhards, 2009-07-15 + */ /* prototypes */ PROTOTYPEObj(conf); -/* TODO: remove them below (means move the config init code) -- rgerhards, 2008-02-19 */ -extern EHostnameCmpMode eDfltHostnameCmpMode; -extern cstr_t *pDfltHostnameCmp; -extern cstr_t *pDfltProgNameCmp; +/* TODO: the following 2 need to go in conf obj interface... */ +rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); +rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); + #endif /* #ifndef INCLUDED_CONF_H */ diff --git a/runtime/ctok.c b/runtime/ctok.c index d2cd8bbd..18ddaed2 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -87,11 +87,12 @@ ctokUngetCharFromStream(ctok_t *pThis, uchar __attribute__((unused)) c) } -/* get the next character from the input "stream" (currently just a in-memory - * string...) -- rgerhards, 2008-02-19 +/* get the next character from the input "stream". Note that this version + * does NOT look for comment characters as end-of-stream, so it is suitable + * when building constant strings! -- rgerhards, 2010-03-01 */ -static rsRetVal -ctokGetCharFromStream(ctok_t *pThis, uchar *pc) +static inline rsRetVal +ctokGetCharFromStreamNoComment(ctok_t *pThis, uchar *pc) { DEFiRet; @@ -99,7 +100,7 @@ ctokGetCharFromStream(ctok_t *pThis, uchar *pc) ASSERT(pc != NULL); /* end of string or begin of comment terminates the "stream" */ - if(*pThis->pp == '\0' || *pThis->pp == '#') { + if(*pThis->pp == '\0') { ABORT_FINALIZE(RS_RET_EOS); } else { *pc = *pThis->pp; @@ -111,6 +112,28 @@ finalize_it: } +/* get the next character from the input "stream" (currently just a in-memory + * string...) -- rgerhards, 2008-02-19 + */ +static rsRetVal +ctokGetCharFromStream(ctok_t *pThis, uchar *pc) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, ctok); + ASSERT(pc != NULL); + + CHKiRet(ctokGetCharFromStreamNoComment(pThis, pc)); + /* begin of comment terminates the "stream"! */ + if(*pc == '#') { + ABORT_FINALIZE(RS_RET_EOS); + } + +finalize_it: + RETiRet; +} + + /* skip whitespace in the input "stream". * rgerhards, 2008-02-19 */ @@ -258,18 +281,18 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_MSGVAR; } - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); /* this loop is quite simple, a variable name is terminated when a non-supported * character is detected. Note that we currently permit a numerical digit as the * first char, which is not permitted by ABNF. -- rgerhards, 2009-03-10 */ while(isalpha(c) || isdigit(c) || (c == '_') || (c == '-')) { - CHKiRet(rsCStrAppendChar(pstrVal, tolower(c))); + CHKiRet(cstrAppendChar(pstrVal, tolower(c))); CHKiRet(ctokGetCharFromStream(pThis, &c)); } CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ - CHKiRet(rsCStrFinish(pstrVal)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -277,7 +300,7 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -301,25 +324,25 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_SIMPSTR; - CHKiRet(rsCStrConstruct(&pstrVal)); - CHKiRet(ctokGetCharFromStream(pThis, &c)); + CHKiRet(cstrConstruct(&pstrVal)); + CHKiRet(ctokGetCharFromStreamNoComment(pThis, &c)); /* while we are in escape mode (had a backslash), no sequence * terminates the loop. If outside, it is terminated by a single quote. */ while(bInEsc || c != '\'') { if(bInEsc) { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); bInEsc = 0; } else { if(c == '\\') { bInEsc = 1; } else { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); } } - CHKiRet(ctokGetCharFromStream(pThis, &c)); + CHKiRet(ctokGetCharFromStreamNoComment(pThis, &c)); } - CHKiRet(rsCStrFinish(pStrB)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -327,7 +350,7 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -519,8 +542,9 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) CHKiRet(ctokUngetCharFromStream(pThis, c)); pToken->tok = ctok_FUNCTION; /* fill function name */ - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); CHKiRet(rsCStrSetSzStr(pstrVal, szWord)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); } else { /* give up... */ dbgprintf("parser has an invalid word (token) '%s'\n", szWord); diff --git a/runtime/datetime.c b/runtime/datetime.c index 19e61a0a..47d7ac0e 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -49,6 +49,8 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +/* the following table of ten powers saves us some computation */ +static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; /* ------------------------------ methods ------------------------------ */ @@ -121,9 +123,28 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) t->OffsetMode = '+'; t->OffsetHour = lBias / 3600; t->OffsetMinute = lBias % 3600; + t->timeType = TIME_TYPE_RFC5424; /* we have a high precision timestamp */ } +/* A fast alternative to getCurrTime() and time() that only obtains + * a timestamp like time() does. I was told that gettimeofday(), at + * least under Linux, is much faster than time() and I could confirm + * this testing. So I created that function as a replacement. + * rgerhards, 2009-11-12 + */ +static time_t +getTime(time_t *ttSeconds) +{ + struct timeval tp; + + if(gettimeofday(&tp, NULL) == -1) + return -1; + + if(ttSeconds != NULL) + *ttSeconds = tp.tv_sec; + return tp.tv_sec; +} /******************************************************************* @@ -140,6 +161,7 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) * DO NOT PUT ANY OTHER CODE IN THIS BEGIN ... END BLOCK!!!! */ + /** * Parse a 32 bit integer number from a string. * @@ -147,18 +169,21 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) * must be positioned at the first digit. Will be updated * so that on return it points to the first character AFTER * the integer parsed. + * \param pLenStr pointer to string length, decremented on exit by + * characters processed + * Note that if an empty string (len < 1) is passed in, + * the method always returns zero. * \retval The number parsed. */ - -static int srSLMGParseInt32(uchar** ppsz) +static int srSLMGParseInt32(uchar** ppsz, int *pLenStr) { - int i; + register int i; i = 0; - while(isdigit((int) **ppsz)) - { + while(*pLenStr > 0 && isdigit((int) **ppsz)) { i = i * 10 + **ppsz - '0'; ++(*ppsz); + --(*pLenStr); } return i; @@ -170,9 +195,13 @@ static int srSLMGParseInt32(uchar** ppsz) * updates the parse pointer position. The pTime parameter * is guranteed to be updated only if a new valid timestamp * could be obtained (restriction added 2008-09-16 by rgerhards). + * This method now also checks the maximum string length it is passed. + * If a *valid* timestamp is found, the string length is decremented + * by the number of characters processed. If it is not a valid timestamp, + * the length is kept unmodified. -- rgerhards, 2009-09-23 */ static rsRetVal -ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) +ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS, int *pLenStr) { uchar *pszTS = *ppszTS; /* variables to temporarily hold time information while we parse */ @@ -187,6 +216,7 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) char OffsetMode; /* UTC offset + or - */ char OffsetHour; /* UTC offset in hours */ int OffsetMinute; /* UTC offset in minutes */ + int lenStr; /* end variables to temporarily hold time information while we parse */ DEFiRet; @@ -194,48 +224,55 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) assert(ppszTS != NULL); assert(pszTS != NULL); - year = srSLMGParseInt32(&pszTS); + lenStr = *pLenStr; + year = srSLMGParseInt32(&pszTS, &lenStr); /* We take the liberty to accept slightly malformed timestamps e.g. in * the format of 2003-9-1T1:0:0. This doesn't hurt on receiving. Of course, * with the current state of affairs, we would never run into this code * here because at postion 11, there is no "T" in such cases ;) */ - if(*pszTS++ != '-') + if(lenStr == 0 || *pszTS++ != '-') ABORT_FINALIZE(RS_RET_INVLD_TIME); - month = srSLMGParseInt32(&pszTS); + --lenStr; + month = srSLMGParseInt32(&pszTS, &lenStr); if(month < 1 || month > 12) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != '-') + if(lenStr == 0 || *pszTS++ != '-') ABORT_FINALIZE(RS_RET_INVLD_TIME); - day = srSLMGParseInt32(&pszTS); + --lenStr; + day = srSLMGParseInt32(&pszTS, &lenStr); if(day < 1 || day > 31) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != 'T') + if(lenStr == 0 || *pszTS++ != 'T') ABORT_FINALIZE(RS_RET_INVLD_TIME); + --lenStr; - hour = srSLMGParseInt32(&pszTS); + hour = srSLMGParseInt32(&pszTS, &lenStr); if(hour < 0 || hour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - minute = srSLMGParseInt32(&pszTS); + --lenStr; + minute = srSLMGParseInt32(&pszTS, &lenStr); if(minute < 0 || minute > 59) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - second = srSLMGParseInt32(&pszTS); + --lenStr; + second = srSLMGParseInt32(&pszTS, &lenStr); if(second < 0 || second > 60) ABORT_FINALIZE(RS_RET_INVLD_TIME); /* Now let's see if we have secfrac */ - if(*pszTS == '.') { + if(lenStr > 0 && *pszTS == '.') { + --lenStr; uchar *pszStart = ++pszTS; - secfrac = srSLMGParseInt32(&pszTS); + secfrac = srSLMGParseInt32(&pszTS, &lenStr); secfracPrecision = (int) (pszTS - pszStart); } else { secfracPrecision = 0; @@ -243,23 +280,27 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) } /* check the timezone */ - if(*pszTS == 'Z') - { + if(lenStr == 0) + ABORT_FINALIZE(RS_RET_INVLD_TIME); + + if(*pszTS == 'Z') { + --lenStr; pszTS++; /* eat Z */ OffsetMode = 'Z'; OffsetHour = 0; OffsetMinute = 0; } else if((*pszTS == '+') || (*pszTS == '-')) { OffsetMode = *pszTS; + --lenStr; pszTS++; - OffsetHour = srSLMGParseInt32(&pszTS); + OffsetHour = srSLMGParseInt32(&pszTS, &lenStr); if(OffsetHour < 0 || OffsetHour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - OffsetMinute = srSLMGParseInt32(&pszTS); + OffsetMinute = srSLMGParseInt32(&pszTS, &lenStr); if(OffsetMinute < 0 || OffsetMinute > 59) ABORT_FINALIZE(RS_RET_INVLD_TIME); } else { @@ -268,10 +309,12 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) } /* OK, we actually have a 3339 timestamp, so let's indicated this */ - if(*pszTS == ' ') - ++pszTS; - else - ABORT_FINALIZE(RS_RET_INVLD_TIME); + if(lenStr > 0) { + if(*pszTS != ' ') /* if it is not a space, it can not be a "good" time - 2010-02-22 rgerhards */ + ABORT_FINALIZE(RS_RET_INVLD_TIME); + ++pszTS; /* just skip past it */ + --lenStr; + } /* we had success, so update parse pointer and caller-provided timestamp */ *ppszTS = pszTS; @@ -287,6 +330,7 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) pTime->OffsetMode = OffsetMode; pTime->OffsetHour = OffsetHour; pTime->OffsetMinute = OffsetMinute; + *pLenStr = lenStr; finalize_it: RETiRet; @@ -305,9 +349,13 @@ finalize_it: * permits us to use a pre-aquired timestamp and thus avoids to do * a (costly) time() call. Thanks to David Lang for insisting on * time() call reduction ;). + * This method now also checks the maximum string length it is passed. + * If a *valid* timestamp is found, the string length is decremented + * by the number of characters processed. If it is not a valid timestamp, + * the length is kept unmodified. -- rgerhards, 2009-09-23 */ static rsRetVal -ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) +ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS, int *pLenStr) { /* variables to temporarily hold time information while we parse */ int month; @@ -317,6 +365,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) int minute; int second; /* end variables to temporarily hold time information while we parse */ + int lenStr; uchar *pszTS; DEFiRet; @@ -324,6 +373,8 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) pszTS = *ppszTS; assert(pszTS != NULL); assert(pTime != NULL); + assert(pLenStr != NULL); + lenStr = *pLenStr; /* If we look at the month (Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec), * we may see the following character sequences occur: @@ -333,6 +384,10 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) * We will use this for parsing, as it probably is the * fastest way to parse it. * + * 2009-08-17: we now do case-insensitive comparisons, as some devices obviously do not + * obey to the RFC-specified case. As we need to guess in any case, we can ignore case + * in the first place -- rgerhards + * * 2005-07-18, well sometimes it pays to be a bit more verbose, even in C... * Fixed a bug that lead to invalid detection of the data. The issue was that * we had an if(++pszTS == 'x') inside of some of the consturcts below. However, @@ -342,22 +397,26 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) * june, when it first manifested. This also lead to invalid parsing of the rest * of the message, as the time stamp was not detected to be correct. - rgerhards */ + if(lenStr < 3) + ABORT_FINALIZE(RS_RET_INVLD_TIME); + switch(*pszTS++) { + case 'j': case 'J': - if(*pszTS == 'a') { + if(*pszTS == 'a' || *pszTS == 'A') { ++pszTS; - if(*pszTS == 'n') { + if(*pszTS == 'n' || *pszTS == 'N') { ++pszTS; month = 1; } else ABORT_FINALIZE(RS_RET_INVLD_TIME); - } else if(*pszTS == 'u') { + } else if(*pszTS == 'u' || *pszTS == 'U') { ++pszTS; - if(*pszTS == 'n') { + if(*pszTS == 'n' || *pszTS == 'N') { ++pszTS; month = 6; - } else if(*pszTS == 'l') { + } else if(*pszTS == 'l' || *pszTS == 'L') { ++pszTS; month = 7; } else @@ -365,10 +424,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'f': case 'F': - if(*pszTS == 'e') { + if(*pszTS == 'e' || *pszTS == 'E') { ++pszTS; - if(*pszTS == 'b') { + if(*pszTS == 'b' || *pszTS == 'B') { ++pszTS; month = 2; } else @@ -376,13 +436,14 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'm': case 'M': - if(*pszTS == 'a') { + if(*pszTS == 'a' || *pszTS == 'A') { ++pszTS; - if(*pszTS == 'r') { + if(*pszTS == 'r' || *pszTS == 'R') { ++pszTS; month = 3; - } else if(*pszTS == 'y') { + } else if(*pszTS == 'y' || *pszTS == 'Y') { ++pszTS; month = 5; } else @@ -390,17 +451,18 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'a': case 'A': - if(*pszTS == 'p') { + if(*pszTS == 'p' || *pszTS == 'P') { ++pszTS; - if(*pszTS == 'r') { + if(*pszTS == 'r' || *pszTS == 'R') { ++pszTS; month = 4; } else ABORT_FINALIZE(RS_RET_INVLD_TIME); - } else if(*pszTS == 'u') { + } else if(*pszTS == 'u' || *pszTS == 'U') { ++pszTS; - if(*pszTS == 'g') { + if(*pszTS == 'g' || *pszTS == 'G') { ++pszTS; month = 8; } else @@ -408,10 +470,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 's': case 'S': - if(*pszTS == 'e') { + if(*pszTS == 'e' || *pszTS == 'E') { ++pszTS; - if(*pszTS == 'p') { + if(*pszTS == 'p' || *pszTS == 'P') { ++pszTS; month = 9; } else @@ -419,10 +482,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'o': case 'O': - if(*pszTS == 'c') { + if(*pszTS == 'c' || *pszTS == 'C') { ++pszTS; - if(*pszTS == 't') { + if(*pszTS == 't' || *pszTS == 'T') { ++pszTS; month = 10; } else @@ -430,10 +494,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'n': case 'N': - if(*pszTS == 'o') { + if(*pszTS == 'o' || *pszTS == 'O') { ++pszTS; - if(*pszTS == 'v') { + if(*pszTS == 'v' || *pszTS == 'V') { ++pszTS; month = 11; } else @@ -441,10 +506,11 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) } else ABORT_FINALIZE(RS_RET_INVLD_TIME); break; + case 'd': case 'D': - if(*pszTS == 'e') { + if(*pszTS == 'e' || *pszTS == 'E') { ++pszTS; - if(*pszTS == 'c') { + if(*pszTS == 'c' || *pszTS == 'C') { ++pszTS; month = 12; } else @@ -456,26 +522,32 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) ABORT_FINALIZE(RS_RET_INVLD_TIME); } + lenStr -= 3; + /* done month */ - if(*pszTS++ != ' ') + if(lenStr == 0 || *pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); + --lenStr; /* we accept a slightly malformed timestamp when receiving. This is * we accept one-digit days */ - if(*pszTS == ' ') + if(*pszTS == ' ') { + --lenStr; ++pszTS; + } - day = srSLMGParseInt32(&pszTS); + day = srSLMGParseInt32(&pszTS, &lenStr); if(day < 1 || day > 31) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ' ') + if(lenStr == 0 || *pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); + --lenStr; /* time part */ - hour = srSLMGParseInt32(&pszTS); + hour = srSLMGParseInt32(&pszTS, &lenStr); if(hour > 1970 && hour < 2100) { /* if so, we assume this actually is a year. This is a format found * e.g. in Cisco devices. @@ -485,23 +557,26 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) year = hour; /* re-query the hour, this time it must be valid */ - if(*pszTS++ != ' ') + if(lenStr == 0 || *pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); - hour = srSLMGParseInt32(&pszTS); + --lenStr; + hour = srSLMGParseInt32(&pszTS, &lenStr); } if(hour < 0 || hour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - minute = srSLMGParseInt32(&pszTS); + --lenStr; + minute = srSLMGParseInt32(&pszTS, &lenStr); if(minute < 0 || minute > 59) ABORT_FINALIZE(RS_RET_INVLD_TIME); - if(*pszTS++ != ':') + if(lenStr == 0 || *pszTS++ != ':') ABORT_FINALIZE(RS_RET_INVLD_TIME); - second = srSLMGParseInt32(&pszTS); + --lenStr; + second = srSLMGParseInt32(&pszTS, &lenStr); if(second < 0 || second > 60) ABORT_FINALIZE(RS_RET_INVLD_TIME); @@ -509,8 +584,16 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) * invalid format, it occurs frequently enough (e.g. with Cisco devices) * to permit it as a valid case. -- rgerhards, 2008-09-12 */ - if(*pszTS++ == ':') + if(lenStr > 0 && *pszTS == ':') { ++pszTS; /* just skip past it */ + --lenStr; + } + if(lenStr > 0) { + if(*pszTS != ' ') /* if it is not a space, it can not be a "good" time - 2010-02-22 rgerhards */ + ABORT_FINALIZE(RS_RET_INVLD_TIME); + ++pszTS; /* just skip past it */ + --lenStr; + } /* we had success, so update parse pointer and caller-provided timestamp * fields we do not have are not updated in the caller's timestamp. This @@ -527,6 +610,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) pTime->second = second; pTime->secfracPrecision = 0; pTime->secfrac = 0; + *pLenStr = lenStr; finalize_it: RETiRet; @@ -545,7 +629,7 @@ finalize_it: * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) +int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf) { /* currently we do not consider localtime/utc. This may later be * added. If so, I recommend using a property replacer option @@ -554,28 +638,54 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) * rgerhards, 2007-06-26 */ assert(ts != NULL); - assert(pDst != NULL); - - if (iLenDst < 15) /* we need at least 14 bytes - 14 digits for timestamp + '\n' */ - return(0); + assert(pBuf != NULL); - return(snprintf(pDst, iLenDst, "%4.4d%2.2d%2.2d%2.2d%2.2d%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = (ts->month / 10) % 10 + '0'; + pBuf[5] = ts->month % 10 + '0'; + pBuf[6] = (ts->day / 10) % 10 + '0'; + pBuf[7] = ts->day % 10 + '0'; + pBuf[8] = (ts->hour / 10) % 10 + '0'; + pBuf[9] = ts->hour % 10 + '0'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = (ts->second / 10) % 10 + '0'; + pBuf[13] = ts->second % 10 + '0'; + pBuf[14] = '\0'; + return 15; } -int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) +int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf) { - /* see note in formatTimestampToMySQL, applies here as well */ - assert(ts != NULL); - assert(pDst != NULL); - - if (iLenDst < 21) /* we need 20 bytes + '\n' */ - return(0); + /* see note in formatTimestampToMySQL, applies here as well */ + assert(ts != NULL); + assert(pBuf != NULL); - return(snprintf(pDst, iLenDst, "%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = ' '; + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = ':'; + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = ':'; + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + pBuf[19] = '\0'; + return 19; } @@ -585,35 +695,36 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) * buffer that will receive the resulting string. The function * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. - * The buffer must be at least 10 bytes large. + * The buffer must be at least 7 bytes large. * rgerhards, 2008-06-06 */ -int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf) { - int lenRet; - char szFmtStr[64]; + int iBuf; + int power; + int secfrac; + short digit; assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf >= 10); + iBuf = 0; if(ts->secfracPrecision > 0) - { /* We must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual formating. - */ - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), "%%0%dd", ts->secfracPrecision); - lenRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->secfrac); + { + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } } else { - pBuf[0] = '0'; - pBuf[1] = '\0'; - lenRet = 1; + pBuf[iBuf++] = '0'; } + pBuf[iBuf] = '\0'; - return(lenRet); + return iBuf; } @@ -625,48 +736,73 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestamp3339(struct syslogTime *ts, char* pBuf) { - int iRet; - char szTZ[7]; /* buffer for TZ information */ + int iBuf; + int power; + int secfrac; + short digit; + BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - - if(iLenBuf < 20) - return(0); /* we NEED at least 20 bytes */ - /* do TZ information first, this is easier to take care of "Z" zone in rfc3339 */ + /* start with fixed parts */ + /* year yyyy */ + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + /* month */ + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + /* day */ + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = 'T'; + /* hour */ + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = ':'; + /* minute */ + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = ':'; + /* second */ + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + + iBuf = 19; /* points to next free entry, now it becomes dynamic! */ + + if(ts->secfracPrecision > 0) { + pBuf[iBuf++] = '.'; + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } + } + if(ts->OffsetMode == 'Z') { - szTZ[0] = 'Z'; - szTZ[1] = '\0'; + pBuf[iBuf++] = 'Z'; } else { - snprintf(szTZ, sizeof(szTZ) / sizeof(char), "%c%2.2d:%2.2d", - ts->OffsetMode, ts->OffsetHour, ts->OffsetMinute); + pBuf[iBuf++] = ts->OffsetMode; + pBuf[iBuf++] = (ts->OffsetHour / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetHour % 10 + '0'; + pBuf[iBuf++] = ':'; + pBuf[iBuf++] = (ts->OffsetMinute / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetMinute % 10 + '0'; } - if(ts->secfracPrecision > 0) - { /* we now need to include fractional seconds. While doing so, we must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual - * formating (mmmmhhh... kind of self-modifying code... ;)). - */ - char szFmtStr[64]; - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), - "%%04d-%%02d-%%02dT%%02d:%%02d:%%02d.%%0%dd%%s", - ts->secfracPrecision); - iRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, ts->secfrac, szTZ); - } - else - iRet = snprintf(pBuf, iLenBuf, - "%4.4d-%2.2d-%2.2dT%2.2d:%2.2d:%2.2d%s", - ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, szTZ); - return(iRet); + pBuf[iBuf] = '\0'; + + ENDfunc + return iBuf; } /** @@ -676,46 +812,35 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) * returns the size of the timestamp written in bytes (without * the string termnator). If 0 is returend, an error occured. */ -int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestamp3164(struct syslogTime *ts, char* pBuf) { - static char* monthNames[13] = {"ERR", "Jan", "Feb", "Mar", - "Apr", "May", "Jun", "Jul", - "Aug", "Sep", "Oct", "Nov", "Dec"}; + static char* monthNames[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + int iDay; assert(ts != NULL); assert(pBuf != NULL); - if(iLenBuf < 16) - return(0); /* we NEED 16 bytes */ - return(snprintf(pBuf, iLenBuf, "%s %2d %2.2d:%2.2d:%2.2d", - monthNames[ts->month], ts->day, ts->hour, - ts->minute, ts->second - )); + pBuf[0] = monthNames[(ts->month - 1)% 12][0]; + pBuf[1] = monthNames[(ts->month - 1) % 12][1]; + pBuf[2] = monthNames[(ts->month - 1) % 12][2]; + pBuf[3] = ' '; + iDay = (ts->day / 10) % 10; /* we need to write a space if the first digit is 0 */ + pBuf[4] = iDay ? iDay + '0' : ' '; + pBuf[5] = ts->day % 10 + '0'; + pBuf[6] = ' '; + pBuf[7] = (ts->hour / 10) % 10 + '0'; + pBuf[8] = ts->hour % 10 + '0'; + pBuf[9] = ':'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = ':'; + pBuf[13] = (ts->second / 10) % 10 + '0'; + pBuf[14] = ts->second % 10 + '0'; + pBuf[15] = '\0'; + return 16; /* traditional: number of bytes written */ } -/** - * Format a syslogTimestamp to a text format. - * The caller must provide the timestamp as well as a character - * buffer that will receive the resulting string. The function - * returns the size of the timestamp written in bytes (without - * the string termnator). If 0 is returend, an error occured. - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int formatTimestamp(struct syslogTime *ts, char* pBuf, size_t iLenBuf) -{ - assert(ts != NULL); - assert(pBuf != NULL); - - if(ts->timeType == 1) { - return(formatTimestamp3164(ts, pBuf, iLenBuf)); - } - - if(ts->timeType == 2) { - return(formatTimestamp3339(ts, pBuf, iLenBuf)); - } - return(0); -} -#endif /* queryInterface function * rgerhards, 2008-03-05 */ @@ -731,6 +856,7 @@ CODESTARTobjQueryInterface(datetime) * of course, also affects the "if" above). */ pIf->getCurrTime = getCurrTime; + pIf->GetTime = getTime; pIf->ParseTIMESTAMP3339 = ParseTIMESTAMP3339; pIf->ParseTIMESTAMP3164 = ParseTIMESTAMP3164; pIf->formatTimestampToMySQL = formatTimestampToMySQL; @@ -749,7 +875,6 @@ ENDobjQueryInterface(datetime) BEGINAbstractObjClassInit(datetime, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); - ENDObjClassInit(datetime) /* vi:set ai: diff --git a/runtime/datetime.h b/runtime/datetime.h index efb0a0af..1de3db95 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -23,8 +23,6 @@ #ifndef INCLUDED_DATETIME_H #define INCLUDED_DATETIME_H -#include "datetime.h" - /* TODO: define error codes */ #define NO_ERRCODE -1 @@ -36,15 +34,17 @@ typedef struct datetime_s { /* interfaces */ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); - rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS); - rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS); - int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst); - int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst, size_t iLenDst); - int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); - int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); - int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); + rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS, int*); + rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS, int*); + int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst); + int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst); + int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf); + int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf); + int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); + /* v3, 2009-11-12 */ + time_t (*GetTime)(time_t *ttSeconds); ENDinterface(datetime) -#define datetimeCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define datetimeCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ /* interface changes: * 1 - initial version * 2 - not compatible to 1 - bugfix required ParseTIMESTAMP3164 to accept char ** as diff --git a/runtime/debug.c b/runtime/debug.c index 4ee90226..6d82397f 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -47,10 +47,14 @@ #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif #include "rsyslog.h" #include "debug.h" #include "atomic.h" +#include "cfsysline.h" #include "obj.h" @@ -732,6 +736,8 @@ static void dbgGetThrdName(char *pszBuf, size_t lenBuf, pthread_t thrd, int bInc */ void dbgSetThrdName(uchar *pszName) { +return; + dbgThrdInfo_t *pThrd = dbgGetThrdInfo(); if(pThrd->pszThrdName != NULL) free(pThrd->pszThrdName); @@ -776,7 +782,7 @@ static void dbgCallStackPrint(dbgThrdInfo_t *pThrd) /* print all threads call stacks */ -static void dbgCallStackPrintAll(void) +void dbgCallStackPrintAll(void) { dbgThrdInfo_t *pThrd; /* stack info */ @@ -828,34 +834,22 @@ sigsegvHdlr(int signum) abort(); } -#if 1 -#pragma GCC diagnostic ignored "-Wempty-body" -/* write the debug message. This is a helper to dbgprintf and dbgoprint which - * contains common code. added 2008-09-26 rgerhards +/* actually write the debug message. This is a separate fuction because the cleanup_push/_pop + * interface otherwise is unsafe to use (generates compiler warnings at least). + * 2009-05-20 rgerhards */ -static void -dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +static inline void +do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg) { static pthread_t ptLastThrdID = 0; static int bWasNL = 0; char pszThrdName[64]; /* 64 is to be on the safe side, anything over 20 is bad... */ - char pszWriteBuf[1024]; + char pszWriteBuf[32*1024]; size_t lenWriteBuf; struct timespec t; - uchar *pszObjName = NULL; - - /* we must get the object name before we lock the mutex, because the object - * potentially calls back into us. If we locked the mutex, we would deadlock - * ourselfs. On the other hand, the GetName call needs not to be protected, as - * this thread has a valid reference. If such an object is deleted by another - * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 - */ - if(pObj != NULL) { - pszObjName = obj.GetName(pObj); - } - - pthread_mutex_lock(&mutdbgprint); - pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif /* The bWasNL handler does not really work. It works if no thread * switching occurs during non-NL messages. Else, things are messed @@ -881,7 +875,14 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(bWasNL) { if(bPrintTime) { +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%4.4ld.%9.9ld:", (long) (t.tv_sec % 10000), t.tv_nsec); if(stddbg != -1) write(stddbg, pszWriteBuf, lenWriteBuf); @@ -903,11 +904,35 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(altdbg != -1) write(altdbg, pszMsg, lenMsg); bWasNL = (pszMsg[lenMsg - 1] == '\n') ? 1 : 0; +} + +#pragma GCC diagnostic ignored "-Wempty-body" +/* write the debug message. This is a helper to dbgprintf and dbgoprint which + * contains common code. added 2008-09-26 rgerhards + */ +static void +dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +{ + uchar *pszObjName = NULL; + + /* we must get the object name before we lock the mutex, because the object + * potentially calls back into us. If we locked the mutex, we would deadlock + * ourselfs. On the other hand, the GetName call needs not to be protected, as + * this thread has a valid reference. If such an object is deleted by another + * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 + */ + if(pObj != NULL) { + pszObjName = obj.GetName(pObj); + } + + pthread_mutex_lock(&mutdbgprint); + pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); + + do_dbgprint(pszObjName, pszMsg, lenMsg); pthread_cleanup_pop(1); } #pragma GCC diagnostic warning "-Wempty-body" -#endif /* print some debug output when an object is given * This is mostly a copy of dbgprintf, but I do not know how to combine it @@ -949,7 +974,7 @@ void dbgprintf(char *fmt, ...) { va_list ap; - char pszWriteBuf[1024]; + char pszWriteBuf[32*1024]; size_t lenWriteBuf; if(!(Debug && debugging_on)) @@ -958,6 +983,16 @@ dbgprintf(char *fmt, ...) va_start(ap, fmt); lenWriteBuf = vsnprintf(pszWriteBuf, sizeof(pszWriteBuf), fmt, ap); va_end(ap); + + if(lenWriteBuf >= sizeof(pszWriteBuf)) { + /* if we need to truncate, do it in a somewhat useful way... */ + pszWriteBuf[sizeof(pszWriteBuf) - 5] = '!'; + pszWriteBuf[sizeof(pszWriteBuf) - 4] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 3] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 2] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 1] = '\n'; + lenWriteBuf = sizeof(pszWriteBuf); + } dbgprint(NULL, pszWriteBuf, lenWriteBuf); } @@ -1050,7 +1085,9 @@ int dbgEntrFunc(dbgFuncDB_t **ppFuncDB, const char *file, const char *func, int /* when we reach this point, we have a fully-initialized FuncDB! */ ATOMIC_INC(pFuncDB->nTimesCalled); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) - dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + } if(pThrd->stackPtr >= (int) (sizeof(pThrd->callStack) / sizeof(dbgFuncDB_t*))) { dbgprintf("%s:%d: %s: debug module: call stack for this thread full, suspending call tracking\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); @@ -1080,10 +1117,12 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet) dbgFuncDBPrintActiveMutexes(pFuncDB, "WARNING: mutex still owned by us as we exit function, mutex: ", pthread_self()); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) { - if(iRet == RS_RET_NO_IRET) - dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); - else - dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + if(iRet == RS_RET_NO_IRET) + dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + else + dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + } } pThrd->stackPtr = iStackPtrRestore; if(pThrd->stackPtr < 0) { @@ -1229,6 +1268,20 @@ dbgPrintNameIsInList(const uchar *pName, dbgPrintName_t *pRoot) } +/* this is a special version of malloc that fills the alloced memory with + * HIGHVALUE, as this helps to identify bugs. -- rgerhards, 2009-10-22 + */ +void * +dbgmalloc(size_t size) +{ + void *pRet; + pRet = malloc(size); + if(pRet != NULL) + memset(pRet, 0xff, size); + return pRet; +} + + /* read in the runtime options * rgerhards, 2008-02-28 */ @@ -1261,14 +1314,22 @@ dbgGetRuntimeOptions(void) "NoLogTimestamp\n" "Nostdoout\n" "filetrace=file (may be provided multiple times)\n" + "DebugOnDemand - enables debugging on USR1, but does not turn on output\n" "\nSee debug.html in your doc set or http://www.rsyslog.com for details\n"); exit(1); } else if(!strcasecmp((char*)optname, "debug")) { /* this is earlier in the process than the -d option, as such it * allows us to spit out debug messages from the very beginning. */ - Debug = 1; + Debug = DEBUG_FULL; + debugging_on = 1; + } else if(!strcasecmp((char*)optname, "debugondemand")) { + /* Enables debugging, but turns off debug output */ + Debug = DEBUG_ONDEMAND; debugging_on = 1; + dbgprintf("Note: debug on demand turned on via configuraton file, " + "use USR1 signal to activate.\n"); + debugging_on = 0; } else if(!strcasecmp((char*)optname, "logfuncflow")) { bLogFuncFlow = 1; } else if(!strcasecmp((char*)optname, "logallocfree")) { diff --git a/runtime/debug.h b/runtime/debug.h index 1375493d..c011dd2d 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -3,7 +3,7 @@ * Definitions for the debug and run-time analysis support module. * Contains a lot of macros. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -29,6 +29,11 @@ #include <pthread.h> #include "obj-types.h" +/* some settings for various debug modes */ +#define DEBUG_OFF 0 +#define DEBUG_ONDEMAND 1 +#define DEBUG_FULL 2 + /* external static data elements (some time to be replaced) */ extern int Debug; /* debug flag - read-only after startup */ extern int debugging_on; /* read-only, except on sig USR1 */ @@ -99,6 +104,7 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet); void dbgSetExecLocation(int iStackPtr, int line); void dbgSetThrdName(uchar *pszName); void dbgPrintAllDebugInfo(void); +void *dbgmalloc(size_t size); /* macros */ #define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } @@ -126,6 +132,12 @@ void dbgPrintAllDebugInfo(void); # define RUNLOG_STR(str) #endif +#ifdef MEMCHECK +# define MALLOC(x) dbgmalloc(x) +#else +# define MALLOC(x) malloc(x) +#endif + /* mutex operations */ #define MUTOP_LOCKWAIT 1 #define MUTOP_LOCK 2 @@ -134,8 +146,7 @@ void dbgPrintAllDebugInfo(void); /* debug aides */ -//#ifdef RTINST -#if 0 // temporarily removed for helgrind +#ifdef RTINST #define d_pthread_mutex_lock(x) dbgMutexLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_trylock(x) dbgMutexTryLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_unlock(x) dbgMutexUnlock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) diff --git a/runtime/glbl.c b/runtime/glbl.c index 28f14320..ac08791f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -35,8 +35,11 @@ #include "rsyslog.h" #include "obj.h" +#include "unicode-helper.h" #include "cfsysline.h" #include "glbl.h" +#include "prop.h" +#include "atomic.h" /* some defaults */ #ifndef DFLT_NETSTRM_DRVR @@ -45,6 +48,7 @@ /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(prop) /* static data * For this object, these variables are obviously what makes the "meat" of the @@ -52,13 +56,14 @@ DEFobjStaticHelpers */ static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ -static int bHUPisRestart = 1; /* should SIGHUP cause a full system restart? */ +static int bParseHOSTNAMEandTAG = 1; /* parser modification (based on startup params!) */ static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ static int option_DisallowWarning = 1; /* complain if message from disallowed sender is received */ static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ +static prop_t *propLocalHostName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup */ static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalDomain; /* our local domain name - read-only after startup */ @@ -68,6 +73,10 @@ static uchar *pszDfltNetstrmDrvr = NULL; /* module name of default netstream dri static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm driver */ static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */ static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */ +static int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ +#ifdef USE_UNLIMITED_SELECT +static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */ +#endif /* define a macro for the simple properties' set and get functions @@ -89,9 +98,9 @@ static dataType Get##nameFunc(void) \ return(nameVar); \ } +SIMP_PROP(ParseHOSTNAMEandTAG, bParseHOSTNAMEandTAG, int) SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) SIMP_PROP(PreserveFQDN, bPreserveFQDN, int) -SIMP_PROP(HUPisRestart, bHUPisRestart, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) @@ -100,6 +109,9 @@ SIMP_PROP(DisableDNS, bDisableDNS, int) SIMP_PROP(LocalDomain, LocalDomain, uchar*) SIMP_PROP(StripDomains, StripDomains, char**) SIMP_PROP(LocalHosts, LocalHosts, char**) +#ifdef USE_UNLIMITED_SELECT +SIMP_PROP(FdSetSize, iFdSetSize, int) +#endif SIMP_PROP_SET(LocalFQDNName, LocalFQDNName, uchar*) SIMP_PROP_SET(LocalHostName, LocalHostName, uchar*) @@ -113,6 +125,24 @@ SIMP_PROP_SET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) /* TO #undef SIMP_PROP_GET +/* return global input termination status + * rgerhards, 2009-07-20 + */ +static int GetGlobalInputTermState(void) +{ + return ATOMIC_FETCH_32BIT(bTerminateInputs); +} + + +/* set global termiantion state to "terminate". Note that this is a + * "once in a lifetime" action which can not be undone. -- gerhards, 2009-07-20 + */ +static void SetGlobalInputTermination(void) +{ + ATOMIC_STORE_1_TO_INT(bTerminateInputs); +} + + /* return our local hostname. if it is not set, "[localhost]" is returned */ static uchar* @@ -132,6 +162,45 @@ GetLocalHostName(void) } +/* generate the local hostname property. This must be done after the hostname info + * has been set as well as PreserveFQDN. + * rgerhards, 2009-06-30 + */ +static rsRetVal +GenerateLocalHostNameProperty(void) +{ + DEFiRet; + uchar *pszName; + + if(propLocalHostName != NULL) + prop.Destruct(&propLocalHostName); + + CHKiRet(prop.Construct(&propLocalHostName)); + if(LocalHostName == NULL) + pszName = (uchar*) "[localhost]"; + else { + if(GetPreserveFQDN() == 1) + pszName = LocalFQDNName; + else + pszName = LocalHostName; + } + CHKiRet(prop.SetString(propLocalHostName, pszName, ustrlen(pszName))); + CHKiRet(prop.ConstructFinalize(propLocalHostName)); + +finalize_it: + RETiRet; +} + + +/* return our local hostname as a string property + */ +static prop_t* +GetLocalHostNameProp(void) +{ + return(propLocalHostName); +} + + /* return the current localhost name as FQDN (requires FQDN to be set) * TODO: we should set the FQDN ourselfs in here! */ @@ -197,13 +266,17 @@ CODESTARTobjQueryInterface(glbl) * of course, also affects the "if" above). */ pIf->GetWorkDir = GetWorkDir; + pIf->GenerateLocalHostNameProperty = GenerateLocalHostNameProperty; + pIf->GetLocalHostNameProp = GetLocalHostNameProp; + pIf->SetGlobalInputTermination = SetGlobalInputTermination; + pIf->GetGlobalInputTermState = GetGlobalInputTermState; #define SIMP_PROP(name) \ pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(ParseHOSTNAMEandTAG); SIMP_PROP(PreserveFQDN); - SIMP_PROP(HUPisRestart); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); @@ -217,6 +290,9 @@ CODESTARTobjQueryInterface(glbl) SIMP_PROP(DfltNetstrmDrvrCAF) SIMP_PROP(DfltNetstrmDrvrKeyFile) SIMP_PROP(DfltNetstrmDrvrCertFile) +#ifdef USE_UNLIMITED_SELECT + SIMP_PROP(FdSetSize) +#endif #undef SIMP_PROP finalize_it: ENDobjQueryInterface(glbl) @@ -249,8 +325,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; - bHUPisRestart = 1; bPreserveFQDN = 0; +#ifdef USE_UNLIMITED_SELECT + iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); +#endif return RS_RET_OK; } @@ -262,6 +340,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a */ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + CHKiRet(objUse(prop, CORE_COMPONENT)); /* register config handlers (TODO: we need to implement a way to unregister them) */ CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, NULL, &pszWorkDir, NULL)); @@ -271,7 +350,6 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) @@ -295,6 +373,7 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(LocalHostName); if(LocalFQDNName != NULL) free(LocalFQDNName); + objRelease(prop, CORE_COMPONENT); ENDObjClassExit(glbl) /* vi:set ai: diff --git a/runtime/glbl.h b/runtime/glbl.h index 5bdf4f57..4b4bdf83 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -8,7 +8,7 @@ * Please note that there currently is no glbl.c file as we do not yet * have any implementations. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -32,6 +32,8 @@ #ifndef GLBL_H_INCLUDED #define GLBL_H_INCLUDED +#include "prop.h" + #define glblGetIOBufSize() 4096 /* size of the IO buffer, e.g. for strm class */ /* interfaces */ @@ -42,7 +44,6 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Set##name)(dataType); SIMP_PROP(MaxLine, int) SIMP_PROP(OptimizeUniProc, int) - SIMP_PROP(HUPisRestart, int) SIMP_PROP(PreserveFQDN, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) @@ -57,9 +58,28 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(DfltNetstrmDrvrCAF, uchar*) SIMP_PROP(DfltNetstrmDrvrKeyFile, uchar*) SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*) + /* added v3, 2009-06-30 */ + rsRetVal (*GenerateLocalHostNameProperty)(void); + prop_t* (*GetLocalHostNameProp)(void); + /* added v4, 2009-07-20 */ + int (*GetGlobalInputTermState)(void); + void (*SetGlobalInputTermination)(void); + /* added v5, 2009-11-03 */ + SIMP_PROP(ParseHOSTNAMEandTAG, int) + /* note: v4, v5 are already used by more recent versions, so we need to skip them! */ + /* added v6, 2009-11-16 as part of varmojfekoj's "unlimited select()" patch + * Note that it must be always present, otherwise the interface would have different + * versions depending on compile settings, what is not acceptable. + * Use this property with care, it is only truly available if UNLIMITED_SELECT is enabled + * (I did not yet further investigate the details, because that code hopefully can be removed + * at some later stage). + */ + SIMP_PROP(FdSetSize, int) + /* v7: was neeeded to mean v5+v6 - do NOT add anything else for that version! */ + /* next change is v8! */ #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ diff --git a/runtime/linkedlist.c b/runtime/linkedlist.c index 8f842e43..cc095f6e 100644 --- a/runtime/linkedlist.c +++ b/runtime/linkedlist.c @@ -398,7 +398,7 @@ rsRetVal llExecFunc(linkedList_t *pThis, rsRetVal (*pFunc)(void*, void*), void* */ llCookie = llCookiePrev; } else if (iRet != RS_RET_OK) { - goto finalize_it; + FINALIZE; } llCookiePrev = llCookie; } diff --git a/runtime/module-template.h b/runtime/module-template.h index 6f7d877c..18aad650 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -39,12 +39,16 @@ #define DEF_OMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ - DEFobjCurrIf(obj) + DEFobjCurrIf(obj) \ + static __attribute__((unused)) int bCoreSupportsBatching; #define DEF_IMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ DEFobjCurrIf(obj) #define DEF_LMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA +#define DEF_PMOD_STATIC_DATA \ + DEFobjCurrIf(obj) \ + DEF_MOD_STATIC_DATA /* Macro to define the module type. Each module can only have a single type. If @@ -64,6 +68,7 @@ static rsRetVal modGetType(eModType_t *modType) \ #define MODULE_TYPE_INPUT MODULE_TYPE(eMOD_IN) #define MODULE_TYPE_OUTPUT MODULE_TYPE(eMOD_OUT) +#define MODULE_TYPE_PARSER MODULE_TYPE(eMOD_PARSER) #define MODULE_TYPE_LIB \ DEF_LMOD_STATIC_DATA \ MODULE_TYPE(eMOD_LIB) @@ -160,6 +165,37 @@ static rsRetVal isCompatibleWithFeature(syslogFeature __attribute__((unused)) eF RETiRet;\ } + +/* beginTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINbeginTransaction \ +static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTbeginTransaction /* currently empty, but may be extended */ + +#define ENDbeginTransaction \ + RETiRet;\ +} + + +/* endTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINendTransaction \ +static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTendTransaction /* currently empty, but may be extended */ + +#define ENDendTransaction \ + RETiRet;\ +} + + /* doAction() */ #define BEGINdoAction \ @@ -324,6 +360,29 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = tryResume;\ } + +/* the following definition is queryEtryPt block that must be added + * if an output module supports the transactional interface. + * rgerhards, 2009-04-27 + */ +#define CODEqueryEtryPt_TXIF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "beginTransaction")) {\ + *pEtryPoint = beginTransaction;\ + } else if(!strcmp((char*) name, "endTransaction")) {\ + *pEtryPoint = endTransaction;\ + } + + +/* the following definition is a queryEtryPt block that must be added + * if a non-output module supports "isCompatibleWithFeature". + * rgerhards, 2009-07-20 + */ +#define CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "isCompatibleWithFeature")) {\ + *pEtryPoint = isCompatibleWithFeature;\ + } + + /* the following definition is the standard block for queryEtryPt for INPUT * modules. This can be used if no specific handling (e.g. to cover version * differences) is needed. @@ -345,6 +404,18 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ #define CODEqueryEtryPt_STD_LIB_QUERIES \ CODEqueryEtryPt_STD_MOD_QUERIES +/* the following definition is the standard block for queryEtryPt for PARSER + * modules. This can be used if no specific handling (e.g. to cover version + * differences) is needed. + */ +#define CODEqueryEtryPt_STD_PMOD_QUERIES \ + CODEqueryEtryPt_STD_MOD_QUERIES \ + else if(!strcmp((char*) name, "parse")) {\ + *pEtryPoint = parse;\ + } else if(!strcmp((char*) name, "GetParserName")) {\ + *pEtryPoint = GetParserName;\ + } + /* modInit() * This has an extra parameter, which is the specific name of the modInit * function. That is needed for built-in modules, which must have unique @@ -393,6 +464,32 @@ finalize_it:\ } +/* now come some check functions, which enable a standard way of obtaining feature + * information from the core. feat is the to-be-tested feature and featVar is a + * variable that receives the result (0-not support, 1-supported). + * This must be a macro, so that it is put into the output's code. Otherwise, we + * would need to rely on a library entry point, which is what we intend to avoid ;) + * rgerhards, 2009-04-27 + */ +#define INITChkCoreFeature(featVar, feat) \ +{ \ + rsRetVal MACRO_Ret; \ + rsRetVal (*pQueryCoreFeatureSupport)(int*, unsigned); \ + int bSupportsIt; \ + featVar = 0; \ + MACRO_Ret = pHostQueryEtryPt((uchar*)"queryCoreFeatureSupport", &pQueryCoreFeatureSupport); \ + if(MACRO_Ret == RS_RET_OK) { \ + /* found entry point, so let's see if core supports it */ \ + CHKiRet((*pQueryCoreFeatureSupport)(&bSupportsIt, feat)); \ + if(bSupportsIt) \ + featVar = 1; \ + } else if(MACRO_Ret != RS_RET_ENTRY_POINT_NOT_FOUND) { \ + ABORT_FINALIZE(MACRO_Ret); /* Something else went wrong, what is not acceptable */ \ + } \ +} + + + /* definitions for host API queries */ #define CODEmodInit_QueryRegCFSLineHdlr \ CHKiRet(pHostQueryEtryPt((uchar*)"regCfSysLineHdlr", &omsdRegCFSLineHdlr)); @@ -509,5 +606,31 @@ static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\ } +/* parse() - main entry point of parser modules + */ +#define BEGINparse \ +static rsRetVal parse(msg_t *pMsg)\ +{\ + DEFiRet; + +#define CODESTARTparse \ + assert(pMsg != NULL); + +#define ENDparse \ + RETiRet;\ +} + + +/* function to specify the parser name. This is done via a single command which + * receives a ANSI string as parameter. + */ +#define PARSER_NAME(x) \ +static rsRetVal GetParserName(uchar **ppSz)\ +{\ + *ppSz = UCHAR_CONSTANT(x);\ + return RS_RET_OK;\ +} + + /* vim:set ai: */ diff --git a/runtime/modules.c b/runtime/modules.c index 32ae659f..1af94abc 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -11,7 +11,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -57,10 +57,12 @@ #include "cfsysline.h" #include "modules.h" #include "errmsg.h" +#include "parser.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +DEFobjCurrIf(parser) /* we must ensure that only one thread at one time tries to load or unload * modules, otherwise we may see race conditions. This first came up with @@ -77,6 +79,26 @@ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ uchar *pModDir = NULL; /* read-only after startup */ +/* we provide a set of dummy functions for modules that do not support the + * some interfaces. + * On the commit feature: As the modules do not support it, they commit each message they + * receive, and as such the dummies can always return RS_RET_OK without causing + * harm. This simplifies things as in action processing we do not need to check + * if the transactional entry points exist. + */ +static rsRetVal dummyBeginTransaction() +{ + return RS_RET_OK; +} +static rsRetVal dummyEndTransaction() +{ + return RS_RET_OK; +} +static rsRetVal dummyIsCompatibleWithFeature() +{ + return RS_RET_INCOMPATIBLE; +} + #ifdef DEBUG /* we add some home-grown support to track our users (and detect who does not free us). In * the long term, this should probably be migrated into debug.c (TODO). -- rgerhards, 2008-03-11 @@ -216,19 +238,38 @@ static void moduleDestruct(modInfo_t *pThis) } +/* This enables a module to query the core for specific features. + * rgerhards, 2009-04-22 + */ +static rsRetVal queryCoreFeatureSupport(int *pBool, unsigned uFeat) +{ + DEFiRet; + + if((pBool == NULL)) + ABORT_FINALIZE(RS_RET_PARAM_ERROR); + + *pBool = (uFeat & CORE_FEATURE_BATCHING) ? 1 : 0; + +finalize_it: + RETiRet; +} + + /* The following function is the queryEntryPoint for host-based entry points. * Modules may call it to get access to core interface functions. Please note * that utility functions can be accessed via shared libraries - at least this * is my current shool of thinking. * Please note that the implementation as a query interface allows to take * care of plug-in interface version differences. -- rgerhards, 2007-07-31 + * ... but often it better not to use a new interface. So we now add core + * functions here that a plugin may request. -- rgerhards, 2009-04-22 */ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) { DEFiRet; if((name == NULL) || (pEtryPoint == NULL)) - return RS_RET_PARAM_ERROR; + ABORT_FINALIZE(RS_RET_PARAM_ERROR); if(!strcmp((char*) name, "regCfSysLineHdlr")) { *pEtryPoint = regCfSysLineHdlr; @@ -236,6 +277,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) *pEtryPoint = objGetObjInterface; } else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) { *pEtryPoint = OMSRgetSupportedTplOpts; + } else if(!strcmp((char*) name, "queryCoreFeatureSupport")) { + *pEtryPoint = queryCoreFeatureSupport; } else { *pEtryPoint = NULL; /* to be on the safe side */ ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND); @@ -361,10 +404,13 @@ finalize_it: static rsRetVal doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr) { - DEFiRet; rsRetVal localRet; modInfo_t *pNew = NULL; + uchar *pParserName; + parser_t *pParser; /* used for parser modules */ + rsRetVal (*GetParserName)(uchar**); rsRetVal (*modGetType)(eModType_t *pType); + DEFiRet; assert(modInit != NULL); @@ -383,7 +429,7 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ * can never change in the lifetime of an module. -- rgerhards, 2007-12-14 */ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getType", &modGetType)); - CHKiRet((iRet = (*modGetType)(&pNew->eType)) != RS_RET_OK); + CHKiRet((*modGetType)(&pNew->eType)); dbgprintf("module of type %d being loaded.\n", pNew->eType); /* OK, we know we can successfully work with the module. So we now fill the @@ -392,6 +438,11 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ */ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modGetID", &pNew->modGetID)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modExit", &pNew->modExit)); + localRet = (*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->isCompatibleWithFeature = dummyIsCompatibleWithFeature; + else if(localRet != RS_RET_OK) + ABORT_FINALIZE(localRet); /* ... and now the module-specific interfaces */ switch(pNew->eType) { @@ -399,21 +450,61 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"runInput", &pNew->mod.im.runInput)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"willRun", &pNew->mod.im.willRun)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"afterRun", &pNew->mod.im.afterRun)); + pNew->mod.im.bCanRun = 0; break; case eMOD_OUT: CHKiRet((*pNew->modQueryEtryPt)((uchar*)"freeInstance", &pNew->freeInstance)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"dbgPrintInstInfo", &pNew->dbgPrintInstInfo)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"doAction", &pNew->mod.om.doAction)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct)); - CHKiRet((*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume)); /* try load optional interfaces */ localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) ABORT_FINALIZE(localRet); + + localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->mod.om.beginTransaction = dummyBeginTransaction; + else if(localRet != RS_RET_OK) + ABORT_FINALIZE(localRet); + + localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) { + pNew->mod.om.endTransaction = dummyEndTransaction; + } else if(localRet != RS_RET_OK) { + ABORT_FINALIZE(localRet); + } break; case eMOD_LIB: break; + case eMOD_PARSER: + /* first, we need to obtain the parser object. We could not do that during + * init as that would have caused class bootstrap issues which are not + * absolutely necessary. Note that we can call objUse() multiple times, it + * handles that. + */ + CHKiRet(objUse(parser, CORE_COMPONENT)); + /* here, we create a new parser object */ + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parse", &pNew->mod.pm.parse)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"GetParserName", &GetParserName)); + CHKiRet(GetParserName(&pParserName)); + CHKiRet(parser.Construct(&pParser)); + + /* check some features */ + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticSanitazion); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoSanitazion(pParser, TRUE)); + } + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticPRIParsing); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoPRIParsing(pParser, TRUE)); + } + + CHKiRet(parser.SetName(pParser, pParserName)); + CHKiRet(parser.SetModPtr(pParser, pNew)); + CHKiRet(parser.ConstructFinalize(pParser)); + break; } pNew->pszName = (uchar*) strdup((char*)name); /* we do not care if strdup() fails, we can accept that */ @@ -460,14 +551,42 @@ static void modPrintList(void) case eMOD_LIB: dbgprintf("library"); break; + case eMOD_PARSER: + dbgprintf("parser"); + break; } dbgprintf(" module.\n"); dbgprintf("Entry points:\n"); dbgprintf("\tqueryEtryPt: 0x%lx\n", (unsigned long) pMod->modQueryEtryPt); - dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); - dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); dbgprintf("\tdbgPrintInstInfo: 0x%lx\n", (unsigned long) pMod->dbgPrintInstInfo); dbgprintf("\tfreeInstance: 0x%lx\n", (unsigned long) pMod->freeInstance); + switch(pMod->eType) { + case eMOD_OUT: + dbgprintf("Output Module Entry Points:\n"); + dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); + dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); + dbgprintf("\ttryResume: 0x%lx\n", (unsigned long) pMod->tryResume); + dbgprintf("\tdoHUP: 0x%lx\n", (unsigned long) pMod->doHUP); + dbgprintf("\tBeginTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.beginTransaction == dummyBeginTransaction) ? + 0 : pMod->mod.om.beginTransaction)); + dbgprintf("\tEndTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.endTransaction == dummyEndTransaction) ? + 0 : pMod->mod.om.endTransaction)); + break; + case eMOD_IN: + dbgprintf("Input Module Entry Points\n"); + dbgprintf("\trunInput: 0x%lx\n", (unsigned long) pMod->mod.im.runInput); + dbgprintf("\twillRun: 0x%lx\n", (unsigned long) pMod->mod.im.willRun); + dbgprintf("\tafterRun: 0x%lx\n", (unsigned long) pMod->mod.im.afterRun); + break; + case eMOD_LIB: + break; + case eMOD_PARSER: + dbgprintf("Parser Module Entry Points\n"); + dbgprintf("\tparse: 0x%lx\n", (unsigned long) pMod->mod.pm.parse); + break; + } dbgprintf("\n"); pMod = GetNxt(pMod); /* done, go next */ } @@ -806,6 +925,7 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); /* We have a problem in our reference counting, which leads to this function * being called too early. This usually is no problem, but if we destroy * the mutex object, we get into trouble. So rather than finding the root cause, diff --git a/runtime/modules.h b/runtime/modules.h index 372529ee..62f86ded 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -12,7 +12,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -51,9 +51,10 @@ #define CURR_MOD_IF_VERSION 5 typedef enum eModType_ { - eMOD_IN, /* input module */ - eMOD_OUT, /* output module */ - eMOD_LIB /* library module - this module provides one or many interfaces */ + eMOD_IN = 0, /* input module */ + eMOD_OUT = 1, /* output module */ + eMOD_LIB = 2, /* library module */ + eMOD_PARSER = 3 /* parser module */ } eModType_t; @@ -73,7 +74,7 @@ typedef enum eModLinkType_ { eMOD_LINK_ALL /* special: all linkage types, e.g. for unload */ } eModLinkType_t; -typedef struct modInfo_s { +struct modInfo_s { struct modInfo_s *pPrev; /* support for creating a double linked module list */ struct modInfo_s *pNext; /* support for creating a linked module list */ int iIFVers; /* Interface version of module */ @@ -106,15 +107,21 @@ typedef struct modInfo_s { rsRetVal (*runInput)(thrdInfo_t*); /* function to gather input and submit to queue */ rsRetVal (*willRun)(void); /* function to gather input and submit to queue */ rsRetVal (*afterRun)(thrdInfo_t*); /* function to gather input and submit to queue */ + int bCanRun; /* cached value of whether willRun() succeeded */ } im; struct {/* data for output modules */ /* below: perform the configured action */ + rsRetVal (*beginTransaction)(void*); rsRetVal (*doAction)(uchar**, unsigned, void*); + rsRetVal (*endTransaction)(void*); rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); } om; struct { /* data for library modules */ - } fm; + } lm; + struct { /* data for parser modules */ + rsRetVal (*parse)(msg_t*); + } pm; } mod; void *pModHdlr; /* handler to the dynamic library holding the module */ # ifdef DEBUG @@ -123,7 +130,7 @@ typedef struct modInfo_s { */ modUsr_t *pModUsrRoot; # endif -} modInfo_t; +}; /* interfaces */ BEGINinterface(module) /* name must also be changed in ENDinterface macro! */ @@ -149,6 +156,5 @@ extern uchar *pModDir; /* read-only after startup */ #endif /* #ifndef MODULES_H_INCLUDED */ -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/msg.c b/runtime/msg.c index dbc3c779..629c6f24 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -35,6 +35,11 @@ #include <string.h> #include <assert.h> #include <ctype.h> +#include <sys/socket.h> +#include <netdb.h> +#if HAVE_MALLOC_H +# include <malloc.h> +#endif #include "rsyslog.h" #include "srUtils.h" #include "stringbuf.h" @@ -46,6 +51,9 @@ #include "regexp.h" #include "atomic.h" #include "unicode-helper.h" +#include "ruleset.h" +#include "prop.h" +#include "net.h" /* static data */ DEFobjStaticHelpers @@ -53,57 +61,470 @@ DEFobjCurrIf(var) DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) +DEFobjCurrIf(prop) +DEFobjCurrIf(net) + +static struct { + uchar *pszName; + short lenName; +} syslog_pri_names[192] = { + { UCHAR_CONSTANT("0"), 3}, + { UCHAR_CONSTANT("1"), 3}, + { UCHAR_CONSTANT("2"), 3}, + { UCHAR_CONSTANT("3"), 3}, + { UCHAR_CONSTANT("4"), 3}, + { UCHAR_CONSTANT("5"), 3}, + { UCHAR_CONSTANT("6"), 3}, + { UCHAR_CONSTANT("7"), 3}, + { UCHAR_CONSTANT("8"), 3}, + { UCHAR_CONSTANT("9"), 3}, + { UCHAR_CONSTANT("10"), 4}, + { UCHAR_CONSTANT("11"), 4}, + { UCHAR_CONSTANT("12"), 4}, + { UCHAR_CONSTANT("13"), 4}, + { UCHAR_CONSTANT("14"), 4}, + { UCHAR_CONSTANT("15"), 4}, + { UCHAR_CONSTANT("16"), 4}, + { UCHAR_CONSTANT("17"), 4}, + { UCHAR_CONSTANT("18"), 4}, + { UCHAR_CONSTANT("19"), 4}, + { UCHAR_CONSTANT("20"), 4}, + { UCHAR_CONSTANT("21"), 4}, + { UCHAR_CONSTANT("22"), 4}, + { UCHAR_CONSTANT("23"), 4}, + { UCHAR_CONSTANT("24"), 4}, + { UCHAR_CONSTANT("25"), 4}, + { UCHAR_CONSTANT("26"), 4}, + { UCHAR_CONSTANT("27"), 4}, + { UCHAR_CONSTANT("28"), 4}, + { UCHAR_CONSTANT("29"), 4}, + { UCHAR_CONSTANT("30"), 4}, + { UCHAR_CONSTANT("31"), 4}, + { UCHAR_CONSTANT("32"), 4}, + { UCHAR_CONSTANT("33"), 4}, + { UCHAR_CONSTANT("34"), 4}, + { UCHAR_CONSTANT("35"), 4}, + { UCHAR_CONSTANT("36"), 4}, + { UCHAR_CONSTANT("37"), 4}, + { UCHAR_CONSTANT("38"), 4}, + { UCHAR_CONSTANT("39"), 4}, + { UCHAR_CONSTANT("40"), 4}, + { UCHAR_CONSTANT("41"), 4}, + { UCHAR_CONSTANT("42"), 4}, + { UCHAR_CONSTANT("43"), 4}, + { UCHAR_CONSTANT("44"), 4}, + { UCHAR_CONSTANT("45"), 4}, + { UCHAR_CONSTANT("46"), 4}, + { UCHAR_CONSTANT("47"), 4}, + { UCHAR_CONSTANT("48"), 4}, + { UCHAR_CONSTANT("49"), 4}, + { UCHAR_CONSTANT("50"), 4}, + { UCHAR_CONSTANT("51"), 4}, + { UCHAR_CONSTANT("52"), 4}, + { UCHAR_CONSTANT("53"), 4}, + { UCHAR_CONSTANT("54"), 4}, + { UCHAR_CONSTANT("55"), 4}, + { UCHAR_CONSTANT("56"), 4}, + { UCHAR_CONSTANT("57"), 4}, + { UCHAR_CONSTANT("58"), 4}, + { UCHAR_CONSTANT("59"), 4}, + { UCHAR_CONSTANT("60"), 4}, + { UCHAR_CONSTANT("61"), 4}, + { UCHAR_CONSTANT("62"), 4}, + { UCHAR_CONSTANT("63"), 4}, + { UCHAR_CONSTANT("64"), 4}, + { UCHAR_CONSTANT("65"), 4}, + { UCHAR_CONSTANT("66"), 4}, + { UCHAR_CONSTANT("67"), 4}, + { UCHAR_CONSTANT("68"), 4}, + { UCHAR_CONSTANT("69"), 4}, + { UCHAR_CONSTANT("70"), 4}, + { UCHAR_CONSTANT("71"), 4}, + { UCHAR_CONSTANT("72"), 4}, + { UCHAR_CONSTANT("73"), 4}, + { UCHAR_CONSTANT("74"), 4}, + { UCHAR_CONSTANT("75"), 4}, + { UCHAR_CONSTANT("76"), 4}, + { UCHAR_CONSTANT("77"), 4}, + { UCHAR_CONSTANT("78"), 4}, + { UCHAR_CONSTANT("79"), 4}, + { UCHAR_CONSTANT("80"), 4}, + { UCHAR_CONSTANT("81"), 4}, + { UCHAR_CONSTANT("82"), 4}, + { UCHAR_CONSTANT("83"), 4}, + { UCHAR_CONSTANT("84"), 4}, + { UCHAR_CONSTANT("85"), 4}, + { UCHAR_CONSTANT("86"), 4}, + { UCHAR_CONSTANT("87"), 4}, + { UCHAR_CONSTANT("88"), 4}, + { UCHAR_CONSTANT("89"), 4}, + { UCHAR_CONSTANT("90"), 4}, + { UCHAR_CONSTANT("91"), 4}, + { UCHAR_CONSTANT("92"), 4}, + { UCHAR_CONSTANT("93"), 4}, + { UCHAR_CONSTANT("94"), 4}, + { UCHAR_CONSTANT("95"), 4}, + { UCHAR_CONSTANT("96"), 4}, + { UCHAR_CONSTANT("97"), 4}, + { UCHAR_CONSTANT("98"), 4}, + { UCHAR_CONSTANT("99"), 4}, + { UCHAR_CONSTANT("100"), 5}, + { UCHAR_CONSTANT("101"), 5}, + { UCHAR_CONSTANT("102"), 5}, + { UCHAR_CONSTANT("103"), 5}, + { UCHAR_CONSTANT("104"), 5}, + { UCHAR_CONSTANT("105"), 5}, + { UCHAR_CONSTANT("106"), 5}, + { UCHAR_CONSTANT("107"), 5}, + { UCHAR_CONSTANT("108"), 5}, + { UCHAR_CONSTANT("109"), 5}, + { UCHAR_CONSTANT("110"), 5}, + { UCHAR_CONSTANT("111"), 5}, + { UCHAR_CONSTANT("112"), 5}, + { UCHAR_CONSTANT("113"), 5}, + { UCHAR_CONSTANT("114"), 5}, + { UCHAR_CONSTANT("115"), 5}, + { UCHAR_CONSTANT("116"), 5}, + { UCHAR_CONSTANT("117"), 5}, + { UCHAR_CONSTANT("118"), 5}, + { UCHAR_CONSTANT("119"), 5}, + { UCHAR_CONSTANT("120"), 5}, + { UCHAR_CONSTANT("121"), 5}, + { UCHAR_CONSTANT("122"), 5}, + { UCHAR_CONSTANT("123"), 5}, + { UCHAR_CONSTANT("124"), 5}, + { UCHAR_CONSTANT("125"), 5}, + { UCHAR_CONSTANT("126"), 5}, + { UCHAR_CONSTANT("127"), 5}, + { UCHAR_CONSTANT("128"), 5}, + { UCHAR_CONSTANT("129"), 5}, + { UCHAR_CONSTANT("130"), 5}, + { UCHAR_CONSTANT("131"), 5}, + { UCHAR_CONSTANT("132"), 5}, + { UCHAR_CONSTANT("133"), 5}, + { UCHAR_CONSTANT("134"), 5}, + { UCHAR_CONSTANT("135"), 5}, + { UCHAR_CONSTANT("136"), 5}, + { UCHAR_CONSTANT("137"), 5}, + { UCHAR_CONSTANT("138"), 5}, + { UCHAR_CONSTANT("139"), 5}, + { UCHAR_CONSTANT("140"), 5}, + { UCHAR_CONSTANT("141"), 5}, + { UCHAR_CONSTANT("142"), 5}, + { UCHAR_CONSTANT("143"), 5}, + { UCHAR_CONSTANT("144"), 5}, + { UCHAR_CONSTANT("145"), 5}, + { UCHAR_CONSTANT("146"), 5}, + { UCHAR_CONSTANT("147"), 5}, + { UCHAR_CONSTANT("148"), 5}, + { UCHAR_CONSTANT("149"), 5}, + { UCHAR_CONSTANT("150"), 5}, + { UCHAR_CONSTANT("151"), 5}, + { UCHAR_CONSTANT("152"), 5}, + { UCHAR_CONSTANT("153"), 5}, + { UCHAR_CONSTANT("154"), 5}, + { UCHAR_CONSTANT("155"), 5}, + { UCHAR_CONSTANT("156"), 5}, + { UCHAR_CONSTANT("157"), 5}, + { UCHAR_CONSTANT("158"), 5}, + { UCHAR_CONSTANT("159"), 5}, + { UCHAR_CONSTANT("160"), 5}, + { UCHAR_CONSTANT("161"), 5}, + { UCHAR_CONSTANT("162"), 5}, + { UCHAR_CONSTANT("163"), 5}, + { UCHAR_CONSTANT("164"), 5}, + { UCHAR_CONSTANT("165"), 5}, + { UCHAR_CONSTANT("166"), 5}, + { UCHAR_CONSTANT("167"), 5}, + { UCHAR_CONSTANT("168"), 5}, + { UCHAR_CONSTANT("169"), 5}, + { UCHAR_CONSTANT("170"), 5}, + { UCHAR_CONSTANT("171"), 5}, + { UCHAR_CONSTANT("172"), 5}, + { UCHAR_CONSTANT("173"), 5}, + { UCHAR_CONSTANT("174"), 5}, + { UCHAR_CONSTANT("175"), 5}, + { UCHAR_CONSTANT("176"), 5}, + { UCHAR_CONSTANT("177"), 5}, + { UCHAR_CONSTANT("178"), 5}, + { UCHAR_CONSTANT("179"), 5}, + { UCHAR_CONSTANT("180"), 5}, + { UCHAR_CONSTANT("181"), 5}, + { UCHAR_CONSTANT("182"), 5}, + { UCHAR_CONSTANT("183"), 5}, + { UCHAR_CONSTANT("184"), 5}, + { UCHAR_CONSTANT("185"), 5}, + { UCHAR_CONSTANT("186"), 5}, + { UCHAR_CONSTANT("187"), 5}, + { UCHAR_CONSTANT("188"), 5}, + { UCHAR_CONSTANT("189"), 5}, + { UCHAR_CONSTANT("190"), 5}, + { UCHAR_CONSTANT("191"), 5} + }; + +/*syslog facility names (as of RFC5424) */ +static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", "syslog", "lpr", + "news", "uucp", "cron", "authpriv", "ftp", "ntp", "audit", + "alert", "clock", "local0", "local1", "local2", "local3", + "local4", "local5", "local6", "local7" }; + +/* table of severity names (in numerical order)*/ +static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" }; + +/* numerical values as string - this is the most efficient approach to convert severity + * and facility values to a numerical string... -- rgerhars, 2009-06-17 + */ -static syslogCODE rs_prioritynames[] = - { - { "alert", LOG_ALERT }, - { "crit", LOG_CRIT }, - { "debug", LOG_DEBUG }, - { "emerg", LOG_EMERG }, - { "err", LOG_ERR }, - { "error", LOG_ERR }, /* DEPRECATED */ - { "info", LOG_INFO }, - { "none", INTERNAL_NOPRI }, /* INTERNAL */ - { "notice", LOG_NOTICE }, - { "panic", LOG_EMERG }, /* DEPRECATED */ - { "warn", LOG_WARNING }, /* DEPRECATED */ - { "warning", LOG_WARNING }, - { NULL, -1 } - }; - -#ifndef LOG_AUTHPRIV -# define LOG_AUTHPRIV LOG_AUTH -#endif -static syslogCODE rs_facilitynames[] = - { - { "auth", LOG_AUTH }, - { "authpriv", LOG_AUTHPRIV }, - { "cron", LOG_CRON }, - { "daemon", LOG_DAEMON }, -#if defined(LOG_FTP) - {"ftp", LOG_FTP}, -#endif - { "kern", LOG_KERN }, - { "lpr", LOG_LPR }, - { "mail", LOG_MAIL }, - { "news", LOG_NEWS }, - { "security", LOG_AUTH }, /* DEPRECATED */ - { "syslog", LOG_SYSLOG }, - { "user", LOG_USER }, - { "uucp", LOG_UUCP }, - { "local0", LOG_LOCAL0 }, - { "local1", LOG_LOCAL1 }, - { "local2", LOG_LOCAL2 }, - { "local3", LOG_LOCAL3 }, - { "local4", LOG_LOCAL4 }, - { "local5", LOG_LOCAL5 }, - { "local6", LOG_LOCAL6 }, - { "local7", LOG_LOCAL7 }, - { NULL, -1 } - }; +static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", + "15", "16", "17", "18", "19", "20", "21", "22", "23" }; /* some forward declarations */ -static int getAPPNAMELen(msg_t *pM); +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); + + +static inline int getProtocolVersion(msg_t *pM) +{ + return(pM->iProtocolVersion); +} + + +/* do a DNS reverse resolution, if not already done, reflect status + * rgerhards, 2009-11-16 + */ +static inline rsRetVal +resolveDNS(msg_t *pMsg) { + rsRetVal localRet; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + DEFiRet; + + CHKiRet(objUse(net, CORE_COMPONENT)); + if(pMsg->msgFlags & NEEDS_DNSRESOL) { + localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); + if(localRet == RS_RET_OK) { + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + } + } +finalize_it: + if(iRet != RS_RET_OK) { + /* best we can do: remove property */ + MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); + prop.Destruct(&propFromHost); + } + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; +} + + +static inline void +getInputName(msg_t *pM, uchar **ppsz, int *plen) +{ + BEGINfunc + if(pM == NULL || pM->pInputName == NULL) { + *ppsz = UCHAR_CONSTANT(""); + *plen = 0; + } else { + prop.GetString(pM->pInputName, ppsz, plen); + } + ENDfunc +} + + +static inline uchar* +getRcvFromIP(msg_t *pM) +{ + uchar *psz; + int len; + BEGINfunc + if(pM == NULL) { + psz = UCHAR_CONSTANT(""); + } else { + resolveDNS(pM); /* make sure we have a resolved entry */ + if(pM->pRcvFromIP == NULL) + psz = UCHAR_CONSTANT(""); + else + prop.GetString(pM->pRcvFromIP, &psz, &len); + } + ENDfunc + return psz; +} + + + +/* map a property name (string) to a property ID */ +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) +{ + uchar *pName; + DEFiRet; + + assert(pCSPropName != NULL); + assert(pPropID != NULL); + pName = rsCStrGetSzStrNoNULL(pCSPropName); + + /* sometimes there are aliases to the original MonitoWare + * property names. These come after || in the ifs below. */ + if(!strcmp((char*) pName, "msg")) { + *pPropID = PROP_MSG; + } else if(!strcmp((char*) pName, "timestamp") + || !strcmp((char*) pName, "timereported")) { + *pPropID = PROP_TIMESTAMP; + } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { + *pPropID = PROP_HOSTNAME; + } else if(!strcmp((char*) pName, "syslogtag")) { + *pPropID = PROP_SYSLOGTAG; + } else if(!strcmp((char*) pName, "rawmsg")) { + *pPropID = PROP_RAWMSG; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(!strcmp((char*) pName, "uxtradmsg")) { + pRes = getUxTradMsg(pMsg); + */ + } else if(!strcmp((char*) pName, "inputname")) { + *pPropID = PROP_INPUTNAME; + } else if(!strcmp((char*) pName, "fromhost")) { + *pPropID = PROP_FROMHOST; + } else if(!strcmp((char*) pName, "fromhost-ip")) { + *pPropID = PROP_FROMHOST_IP; + } else if(!strcmp((char*) pName, "pri")) { + *pPropID = PROP_PRI; + } else if(!strcmp((char*) pName, "pri-text")) { + *pPropID = PROP_PRI_TEXT; + } else if(!strcmp((char*) pName, "iut")) { + *pPropID = PROP_IUT; + } else if(!strcmp((char*) pName, "syslogfacility")) { + *pPropID = PROP_SYSLOGFACILITY; + } else if(!strcmp((char*) pName, "syslogfacility-text")) { + *pPropID = PROP_SYSLOGFACILITY_TEXT; + } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { + *pPropID = PROP_SYSLOGSEVERITY; + } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { + *pPropID = PROP_SYSLOGSEVERITY_TEXT; + } else if(!strcmp((char*) pName, "timegenerated")) { + *pPropID = PROP_TIMEGENERATED; + } else if(!strcmp((char*) pName, "programname")) { + *pPropID = PROP_PROGRAMNAME; + } else if(!strcmp((char*) pName, "protocol-version")) { + *pPropID = PROP_PROTOCOL_VERSION; + } else if(!strcmp((char*) pName, "structured-data")) { + *pPropID = PROP_STRUCTURED_DATA; + } else if(!strcmp((char*) pName, "app-name")) { + *pPropID = PROP_APP_NAME; + } else if(!strcmp((char*) pName, "procid")) { + *pPropID = PROP_PROCID; + } else if(!strcmp((char*) pName, "msgid")) { + *pPropID = PROP_MSGID; + /* here start system properties (those, that do not relate to the message itself */ + } else if(!strcmp((char*) pName, "$now")) { + *pPropID = PROP_SYS_NOW; + } else if(!strcmp((char*) pName, "$year")) { + *pPropID = PROP_SYS_YEAR; + } else if(!strcmp((char*) pName, "$month")) { + *pPropID = PROP_SYS_MONTH; + } else if(!strcmp((char*) pName, "$day")) { + *pPropID = PROP_SYS_DAY; + } else if(!strcmp((char*) pName, "$hour")) { + *pPropID = PROP_SYS_HOUR; + } else if(!strcmp((char*) pName, "$hhour")) { + *pPropID = PROP_SYS_HHOUR; + } else if(!strcmp((char*) pName, "$qhour")) { + *pPropID = PROP_SYS_QHOUR; + } else if(!strcmp((char*) pName, "$minute")) { + *pPropID = PROP_SYS_MINUTE; + } else if(!strcmp((char*) pName, "$myhostname")) { + *pPropID = PROP_SYS_MYHOSTNAME; + } else { + *pPropID = PROP_INVALID; + iRet = RS_RET_VAR_NOT_FOUND; + } + + RETiRet; +} + + +/* map a property ID to a name string (useful for displaying) */ +uchar *propIDToName(propid_t propID) +{ + switch(propID) { + case PROP_MSG: + return UCHAR_CONSTANT("msg"); + case PROP_TIMESTAMP: + return UCHAR_CONSTANT("timestamp"); + case PROP_HOSTNAME: + return UCHAR_CONSTANT("hostname"); + case PROP_SYSLOGTAG: + return UCHAR_CONSTANT("syslogtag"); + case PROP_RAWMSG: + return UCHAR_CONSTANT("rawmsg"); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + case PROP_UXTRADMSG: + pRes = getUxTradMsg(pMsg); + break; + */ + case PROP_INPUTNAME: + return UCHAR_CONSTANT("inputname"); + case PROP_FROMHOST: + return UCHAR_CONSTANT("fromhost"); + case PROP_FROMHOST_IP: + return UCHAR_CONSTANT("fromhost-ip"); + case PROP_PRI: + return UCHAR_CONSTANT("pri"); + case PROP_PRI_TEXT: + return UCHAR_CONSTANT("pri-text"); + case PROP_IUT: + return UCHAR_CONSTANT("iut"); + case PROP_SYSLOGFACILITY: + return UCHAR_CONSTANT("syslogfacility"); + case PROP_SYSLOGFACILITY_TEXT: + return UCHAR_CONSTANT("syslogfacility-text"); + case PROP_SYSLOGSEVERITY: + return UCHAR_CONSTANT("syslogseverity"); + case PROP_SYSLOGSEVERITY_TEXT: + return UCHAR_CONSTANT("syslogseverity-text"); + case PROP_TIMEGENERATED: + return UCHAR_CONSTANT("timegenerated"); + case PROP_PROGRAMNAME: + return UCHAR_CONSTANT("programname"); + case PROP_PROTOCOL_VERSION: + return UCHAR_CONSTANT("protocol-version"); + case PROP_STRUCTURED_DATA: + return UCHAR_CONSTANT("structured-data"); + case PROP_APP_NAME: + return UCHAR_CONSTANT("app-name"); + case PROP_PROCID: + return UCHAR_CONSTANT("procid"); + case PROP_MSGID: + return UCHAR_CONSTANT("msgid"); + case PROP_SYS_NOW: + return UCHAR_CONSTANT("$NOW"); + case PROP_SYS_YEAR: + return UCHAR_CONSTANT("$YEAR"); + case PROP_SYS_MONTH: + return UCHAR_CONSTANT("$MONTH"); + case PROP_SYS_DAY: + return UCHAR_CONSTANT("$DAY"); + case PROP_SYS_HOUR: + return UCHAR_CONSTANT("$HOUR"); + case PROP_SYS_HHOUR: + return UCHAR_CONSTANT("$HHOUR"); + case PROP_SYS_QHOUR: + return UCHAR_CONSTANT("$QHOUR"); + case PROP_SYS_MINUTE: + return UCHAR_CONSTANT("$MINUTE"); + case PROP_SYS_MYHOSTNAME: + return UCHAR_CONSTANT("$MYHOSTNAME"); + default: + return UCHAR_CONSTANT("*invalid property id*"); + } +} + /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we @@ -165,32 +586,9 @@ static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) */ static void MsgPrepareEnqueueLockingCase(msg_t *pThis) { - int iErr; BEGINfunc assert(pThis != NULL); - iErr = pthread_mutexattr_init(&pThis->mutAttr); - if(iErr != 0) { - dbgprintf("error initializing mutex attribute in %s:%d, trying to continue\n", - __FILE__, __LINE__); - } - iErr = pthread_mutexattr_settype(&pThis->mutAttr, PTHREAD_MUTEX_RECURSIVE); - if(iErr != 0) { - dbgprintf("ERROR setting mutex attribute to recursive in %s:%d, trying to continue " - "but we will probably either abort or hang soon\n", - __FILE__, __LINE__); - /* TODO: it makes very little sense to continue here, - * but it requires an iRet interface to gracefully shut - * down. We should do that over time. -- rgerhards, 2008-07-14 - */ - } - pthread_mutex_init(&pThis->mut, &pThis->mutAttr); - - /* we do no longer need the attribute. According to the - * POSIX spec, we can destroy it without affecting the - * initialized mutex (that used the attribute). - * rgerhards, 2008-07-14 - */ - pthread_mutexattr_destroy(&pThis->mutAttr); + pthread_mutex_init(&pThis->mut, NULL); pThis->bDoLock = 1; ENDfunc } @@ -255,6 +653,11 @@ rsRetVal MsgEnableThreadSafety(void) * itself but rather uses a user-supplied value. This enables the caller * to do some tricks to save processing time (done, for example, in the * udp input). + * NOTE: this constructor does NOT call calloc(), as we have many bytes + * inside the structure which do not need to be cleared. bzero() will + * heavily thrash the cache, so we do the init manually (which also + * is the right thing to do with pointers, as they are not neccessarily + * a binary 0 on all machines [but today almost always...]). * rgerhards, 2008-10-06 */ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) @@ -263,14 +666,49 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) msg_t *pM; assert(ppThis != NULL); - if((pM = calloc(1, sizeof(msg_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pM = MALLOC(sizeof(msg_t))); + objConstructSetObjInfo(pM); /* intialize object helper entities */ - /* initialize members that are non-zero */ + /* initialize members in ORDER they appear in structure (think "cache line"!) */ + pM->flowCtlType = 0; + pM->bDoLock = 0; pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; - objConstructSetObjInfo(pM); + pM->offAfterPRI = 0; + pM->offMSG = -1; + pM->iProtocolVersion = 0; + pM->msgFlags = 0; + pM->iLenRawMsg = 0; + pM->iLenMSG = 0; + pM->iLenTAG = 0; + pM->iLenHOSTNAME = 0; + pM->pszRawMsg = NULL; + pM->pszHOSTNAME = NULL; + pM->pszRcvdAt3164 = NULL; + pM->pszRcvdAt3339 = NULL; + pM->pszRcvdAt_MySQL = NULL; + pM->pszRcvdAt_PgSQL = NULL; + pM->pszTIMESTAMP3164 = NULL; + pM->pszTIMESTAMP3339 = NULL; + pM->pszTIMESTAMP_MySQL = NULL; + pM->pszTIMESTAMP_PgSQL = NULL; + pM->pCSProgName = NULL; + pM->pCSStrucData = NULL; + pM->pCSAPPNAME = NULL; + pM->pCSPROCID = NULL; + pM->pCSMSGID = NULL; + pM->pInputName = NULL; + pM->pRcvFromIP = NULL; + pM->rcvFrom.pRcvFrom = NULL; + pM->pRuleset = NULL; + memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); + memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); + pM->TAG.pszTAG = NULL; + pM->pszTimestamp3164[0] = '\0'; + pM->pszTimestamp3339[0] = '\0'; + pM->pszTIMESTAMP_SecFrac[0] = '\0'; + pM->pszRcvdAt_SecFrac[0] = '\0'; /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ @@ -329,6 +767,21 @@ finalize_it: } +/* some free handlers for (slightly) complicated cases... All of them may be called + * with an empty element. + */ +static inline void freeTAG(msg_t *pThis) +{ + if(pThis->iLenTAG >= CONF_TAG_BUFSIZE) + free(pThis->TAG.pszTAG); +} +static inline void freeHOSTNAME(msg_t *pThis) +{ + if(pThis->iLenHOSTNAME >= CONF_HOSTNAME_BUFSIZE) + free(pThis->pszHOSTNAME); +} + + BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ int currRefCount; CODESTARTobjDestruct(msg) @@ -342,29 +795,26 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - free(pThis->pszUxTradMsg); - free(pThis->pszRawMsg); - free(pThis->pszTAG); - free(pThis->pszHOSTNAME); - free(pThis->pszInputName); - free(pThis->pszRcvFrom); - free(pThis->pszRcvFromIP); - free(pThis->pszMSG); - free(pThis->pszFacility); - free(pThis->pszFacilityStr); - free(pThis->pszSeverity); - free(pThis->pszSeverityStr); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); + freeTAG(pThis); + freeHOSTNAME(pThis); + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); + if((pThis->msgFlags & NEEDS_DNSRESOL) == 0) { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } else { + free(pThis->rcvFrom.pfrominet); + } + if(pThis->pRcvFromIP != NULL) + prop.Destruct(&pThis->pRcvFromIP); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); - free(pThis->pszRcvdAt_SecFrac); free(pThis->pszRcvdAt_MySQL); free(pThis->pszRcvdAt_PgSQL); - free(pThis->pszTIMESTAMP3164); - free(pThis->pszTIMESTAMP3339); - free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); - free(pThis->pszPRI); if(pThis->pCSProgName != NULL) rsCStrDestruct(&pThis->pCSProgName); if(pThis->pCSStrucData != NULL) @@ -379,8 +829,29 @@ CODESTARTobjDestruct(msg) MsgUnlock(pThis); # endif funcDeleteMutex(pThis); + /* now we need to do our own optimization. Testing has shown that at least the glibc + * malloc() subsystem returns memory to the OS far too late in our case. So we need + * to help it a bit, by calling malloc_trim(), which will tell the alloc subsystem + * to consolidate and return to the OS. We keep 128K for our use, as a safeguard + * to too-frequent reallocs. But more importantly, we call this hook only every + * 100,000 messages (which is an approximation, as we do not work with atomic + * operations on the counter. --- rgerhards, 2009-06-22. + */ +# if HAVE_MALLOC_TRIM + { /* standard C requires a new block for a new variable definition! + * To simplify matters, we use modulo arithmetic and live with the fact + * that we trim too often when the counter wraps. + */ + static unsigned iTrimCtr = 1; + if(ATOMIC_INC_AND_FETCH(iTrimCtr) % 100000 == 0) { + malloc_trim(128*1024); + } + } +# endif } else { +# ifndef HAVE_ATOMIC_BUILTINS MsgUnlock(pThis); +# endif pThis = NULL; /* tell framework not to destructing the object! */ } ENDobjDestruct(msg) @@ -421,6 +892,7 @@ ENDobjDestruct(msg) msg_t* MsgDup(msg_t* pOld) { msg_t* pNew; + rsRetVal localRet; assert(pOld != NULL); @@ -433,21 +905,63 @@ msg_t* MsgDup(msg_t* pOld) pNew->iRefCount = 1; pNew->iSeverity = pOld->iSeverity; pNew->iFacility = pOld->iFacility; - pNew->bParseHOSTNAME = pOld->bParseHOSTNAME; pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; - tmpCOPYSZ(Severity); - tmpCOPYSZ(SeverityStr); - tmpCOPYSZ(Facility); - tmpCOPYSZ(FacilityStr); - tmpCOPYSZ(PRI); - tmpCOPYSZ(RawMsg); - tmpCOPYSZ(MSG); - tmpCOPYSZ(UxTradMsg); - tmpCOPYSZ(TAG); - tmpCOPYSZ(HOSTNAME); - tmpCOPYSZ(RcvFrom); + pNew->offMSG = pOld->offMSG; + pNew->iLenRawMsg = pOld->iLenRawMsg; + pNew->iLenMSG = pOld->iLenMSG; + pNew->iLenTAG = pOld->iLenTAG; + pNew->iLenHOSTNAME = pOld->iLenHOSTNAME; + if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) { + localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet); + if(localRet != RS_RET_OK) { + /* if something fails, we accept loss of this property, it is + * better than losing the whole message. + */ + pNew->msgFlags &= ~NEEDS_DNSRESOL; + } + } else { + if(pOld->rcvFrom.pRcvFrom != NULL) { + pNew->rcvFrom.pRcvFrom = pOld->rcvFrom.pRcvFrom; + prop.AddRef(pNew->rcvFrom.pRcvFrom); + } + } + if(pOld->pRcvFromIP != NULL) { + pNew->pRcvFromIP = pOld->pRcvFromIP; + prop.AddRef(pNew->pRcvFromIP); + } + if(pOld->pInputName != NULL) { + pNew->pInputName = pOld->pInputName; + prop.AddRef(pNew->pInputName); + } + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + pNew->offAfterPRI = pOld->offAfterPRI; + */ + if(pOld->iLenTAG > 0) { + if(pOld->iLenTAG < CONF_TAG_BUFSIZE) { + memcpy(pNew->TAG.szBuf, pOld->TAG.szBuf, pOld->iLenTAG); + } else { + if((pNew->TAG.pszTAG = srUtilStrDup(pOld->TAG.pszTAG, pOld->iLenTAG)) == NULL) { + msgDestruct(&pNew); + return NULL; + } + pNew->iLenTAG = pOld->iLenTAG; + } + } + if(pOld->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + memcpy(pNew->szRawMsg, pOld->szRawMsg, pOld->iLenRawMsg + 1); + pNew->pszRawMsg = pNew->szRawMsg; + } else { + tmpCOPYSZ(RawMsg); + } + if(pOld->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { + memcpy(pNew->szHOSTNAME, pOld->szHOSTNAME, pOld->iLenHOSTNAME + 1); + pNew->pszHOSTNAME = pNew->szHOSTNAME; + } else { + tmpCOPYSZ(HOSTNAME); + } tmpCOPYCSTR(ProgName); tmpCOPYCSTR(StrucData); @@ -474,17 +988,20 @@ msg_t* MsgDup(msg_t* pOld) * We do not serialize the cache properties. We re-create them when needed. * This saves us a lot of memory. Performance is no concern, as serializing * is a so slow operation that recration of the caches does not count. Also, - * we do not serialize bParseHOSTNAME, as this is only a helper variable + * we do not serialize --currently none--, as this is only a helper variable * during msg construction - and never again used later. * rgerhards, 2008-01-03 */ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) { + uchar *psz; + int len; DEFiRet; assert(pThis != NULL); assert(pStrm != NULL); + /* then serialize elements */ CHKiRet(obj.BeginSerialize(pStrm, (obj_t*) pThis)); objSerializeSCALAR(pStrm, iProtocolVersion, SHORT); objSerializeSCALAR(pStrm, iSeverity, SHORT); @@ -493,21 +1010,33 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + objSerializeSCALAR(pStrm, offsAfterPRI, SHORT); + */ + + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszTAG"), PROPTYPE_PSZ, (void*) + ((pThis->iLenTAG < CONF_TAG_BUFSIZE) ? pThis->TAG.szBuf : pThis->TAG.pszTAG))); objSerializePTR(pStrm, pszRawMsg, PSZ); - objSerializePTR(pStrm, pszMSG, PSZ); - objSerializePTR(pStrm, pszUxTradMsg, PSZ); - objSerializePTR(pStrm, pszTAG, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); - objSerializePTR(pStrm, pszInputName, PSZ); - objSerializePTR(pStrm, pszRcvFrom, PSZ); - objSerializePTR(pStrm, pszRcvFromIP, PSZ); + getInputName(pThis, &psz, &len); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszInputName"), PROPTYPE_PSZ, (void*) psz)); + psz = getRcvFrom(pThis); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFrom"), PROPTYPE_PSZ, (void*) psz)); + psz = getRcvFromIP(pThis); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFromIP"), PROPTYPE_PSZ, (void*) psz)); objSerializePTR(pStrm, pCSStrucData, CSTR); objSerializePTR(pStrm, pCSAPPNAME, CSTR); objSerializePTR(pStrm, pCSPROCID, CSTR); objSerializePTR(pStrm, pCSMSGID, CSTR); + /* offset must be serialized after pszRawMsg, because we need that to obtain the correct + * MSG size. + */ + objSerializeSCALAR(pStrm, offMSG, SHORT); + CHKiRet(obj.EndSerialize(pStrm)); finalize_it: @@ -546,22 +1075,27 @@ msg_t *MsgAddRef(msg_t *pM) * can obtain a PROCID. Take in mind that not every legacy syslog message * actually has a PROCID. * rgerhards, 2005-11-24 + * THIS MUST be called with the message lock locked. */ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) { register int i; + uchar *pszTag; DEFiRet; assert(pM != NULL); + if(pM->pCSPROCID != NULL) return RS_RET_OK; /* we are already done ;) */ if(getProtocolVersion(pM) != 0) return RS_RET_OK; /* we can only emulate if we have legacy format */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); + /* find first '['... */ i = 0; - while((i < pM->iLenTAG) && (pM->pszTAG[i] != '[')) + while((i < pM->iLenTAG) && (pszTag[i] != '[')) ++i; if(!(i < pM->iLenTAG)) return RS_RET_OK; /* no [, so can not emulate... */ @@ -569,10 +1103,9 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) ++i; /* skip '[' */ /* now obtain the PROCID string... */ - CHKiRet(rsCStrConstruct(&pM->pCSPROCID)); - rsCStrSetAllocIncrement(pM->pCSPROCID, 16); - while((i < pM->iLenTAG) && (pM->pszTAG[i] != ']')) { - CHKiRet(rsCStrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); + CHKiRet(cstrConstruct(&pM->pCSPROCID)); + while((i < pM->iLenTAG) && (pszTag[i] != ']')) { + CHKiRet(cstrAppendChar(pM->pCSPROCID, pszTag[i])); ++i; } @@ -582,12 +1115,12 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) * the buffer and simply return. Note that this is NOT an error * case! */ - rsCStrDestruct(&pM->pCSPROCID); + cstrDestruct(&pM->pCSPROCID); FINALIZE; } /* OK, finaally we could obtain a PROCID. So let's use it ;) */ - CHKiRet(rsCStrFinish(pM->pCSPROCID)); + CHKiRet(cstrFinalize(pM->pCSPROCID)); finalize_it: RETiRet; @@ -607,57 +1140,35 @@ finalize_it: * The program name is not parsed by default, because it is infrequently-used. * If it is needed, this function should be called first. It checks if it is * already set and extracts it, if not. - * A message object must be provided, else a crash will occur. + * + * IMPORTANT: A locked message object must be provided, else a crash will occur. * rgerhards, 2005-10-19 */ static rsRetVal aquireProgramName(msg_t *pM) { - DEFiRet; register int i; + uchar *pszTag; + DEFiRet; assert(pM != NULL); if(pM->pCSProgName == NULL) { - /* ok, we do not yet have it. So let's parse the TAG - * to obtain it. - */ - CHKiRet(rsCStrConstruct(&pM->pCSProgName)); - rsCStrSetAllocIncrement(pM->pCSProgName, 33); + /* ok, we do not yet have it. So let's parse the TAG to obtain it. */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); + CHKiRet(cstrConstruct(&pM->pCSProgName)); for( i = 0 - ; (i < pM->iLenTAG) && isprint((int) pM->pszTAG[i]) - && (pM->pszTAG[i] != '\0') && (pM->pszTAG[i] != ':') - && (pM->pszTAG[i] != '[') && (pM->pszTAG[i] != '/') + ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) + && (pszTag[i] != '\0') && (pszTag[i] != ':') + && (pszTag[i] != '[') && (pszTag[i] != '/') ; ++i) { - CHKiRet(rsCStrAppendChar(pM->pCSProgName, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSProgName, pszTag[i])); } - CHKiRet(rsCStrFinish(pM->pCSProgName)); + CHKiRet(cstrFinalize(pM->pCSProgName)); } finalize_it: RETiRet; } -/* This function moves the HOSTNAME inside the message object to the - * TAG. It is a specialised function used to handle the condition when - * a message without HOSTNAME is being processed. The missing HOSTNAME - * is only detected at a later stage, during TAG processing, so that - * we already had set the HOSTNAME property and now need to move it to - * the TAG. Of course, we could do this via a couple of get/set methods, - * but it is far more efficient to do it via this specialised method. - * This is especially important as this can be a very common case, e.g. - * when BSD syslog is acting as a sender. - * rgerhards, 2005-11-10. - */ -void moveHOSTNAMEtoTAG(msg_t *pM) -{ - assert(pM != NULL); - if(pM->pszTAG != NULL) - free(pM->pszTAG); - pM->pszTAG = pM->pszHOSTNAME; - pM->iLenTAG = pM->iLenHOSTNAME; - pM->pszHOSTNAME = NULL; - pM->iLenHOSTNAME = 0; -} - /* Access methods - dumb & easy, not a comment for each ;) */ void setProtocolVersion(msg_t *pM, int iNewVersion) @@ -670,12 +1181,6 @@ void setProtocolVersion(msg_t *pM, int iNewVersion) pM->iProtocolVersion = iNewVersion; } -int getProtocolVersion(msg_t *pM) -{ - assert(pM != NULL); - return(pM->iProtocolVersion); -} - /* note: string is taken from constant pool, do NOT free */ char *getProtocolVersionString(msg_t *pM) { @@ -683,84 +1188,82 @@ char *getProtocolVersionString(msg_t *pM) return(pM->iProtocolVersion ? "1" : "0"); } -int getMSGLen(msg_t *pM) + +static inline void +getRawMsg(msg_t *pM, uchar **pBuf, int *piLen) { - return((pM == NULL) ? 0 : pM->iLenMSG); + if(pM == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + if(pM->pszRawMsg == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + *pBuf = pM->pszRawMsg; + *piLen = pM->iLenRawMsg; + } + } } -char *getRawMsg(msg_t *pM) +/* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 +char *getUxTradMsg(msg_t *pM) { if(pM == NULL) return ""; else - if(pM->pszRawMsg == NULL) - return ""; - else - return (char*)pM->pszRawMsg; + return (char*)pM->pszRawMsg + pM->offAfterPRI; } +*/ -char *getUxTradMsg(msg_t *pM) + +int getMSGLen(msg_t *pM) { - if(pM == NULL) - return ""; - else - if(pM->pszUxTradMsg == NULL) - return ""; - else - return (char*)pM->pszUxTradMsg; + return((pM == NULL) ? 0 : pM->iLenMSG); } -char *getMSG(msg_t *pM) +uchar *getMSG(msg_t *pM) { + uchar *ret; if(pM == NULL) - return ""; - else - if(pM->pszMSG == NULL) - return ""; + ret = UCHAR_CONSTANT(""); + else { + if(pM->iLenMSG == 0) + ret = UCHAR_CONSTANT(""); else - return (char*)pM->pszMSG; + ret = pM->pszRawMsg + pM->offMSG; + } + return ret; } -/* Get PRI value in text form */ -char *getPRI(msg_t *pM) +/* Get PRI value as integer */ +static int getPRIi(msg_t *pM) { - int pri; - BEGINfunc - - if(pM == NULL) - return ""; - - MsgLock(pM); - if(pM->pszPRI == NULL) { - /* OK, we need to construct it... we use a 5 byte buffer - as of - * RFC 3164, it can't be longer. Should it still be, snprintf will truncate... - * Note that we do not use the LOG_MAKEPRI macro. This macro - * is a simple add of the two values under FreeBSD 7. So we implement - * the logic in our own code. This is a change from a bug - * report. -- rgerhards, 2008-07-14 - */ - pri = pM->iFacility * 8 + pM->iSeverity; - if((pM->pszPRI = malloc(5)) == NULL) return ""; - pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); - } - MsgUnlock(pM); - - ENDfunc - return (char*)pM->pszPRI; + return (pM->iFacility << 3) + (pM->iSeverity); } -/* Get PRI value as integer */ -int getPRIi(msg_t *pM) +/* Get PRI value in text form + */ +static inline char *getPRI(msg_t *pM) { - assert(pM != NULL); - return (pM->iFacility << 3) + (pM->iSeverity); + /* PRI is a number in the range 0..191. Thus, we use a simple lookup table to obtain the + * string value. It looks a bit clumpsy here in code ;) + */ + int iPRI; + + if(pM == NULL) + return ""; + + iPRI = getPRIi(pM); + return (iPRI > 191) ? "invld" : (char*)syslog_pri_names[iPRI].pszName; } -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -768,77 +1271,60 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) switch(eFmt) { case tplFmtDefault: + case tplFmtRFC3164Date: MsgLock(pM); if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } - datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); + pM->pszTIMESTAMP3164 = pM->pszTimestamp3164; + datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164); } MsgUnlock(pM); return(pM->pszTIMESTAMP3164); case tplFmtMySQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_MySQL == NULL) { - if((pM->pszTIMESTAMP_MySQL = malloc(15)) == NULL) { + if((pM->pszTIMESTAMP_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToMySQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_MySQL, 15); + datetime.formatTimestampToMySQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_MySQL); } MsgUnlock(pM); return(pM->pszTIMESTAMP_MySQL); case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_PgSQL == NULL) { - if((pM->pszTIMESTAMP_PgSQL = malloc(21)) == NULL) { + if((pM->pszTIMESTAMP_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToPgSQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_PgSQL, 21); + datetime.formatTimestampToPgSQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_PgSQL); } MsgUnlock(pM); return(pM->pszTIMESTAMP_PgSQL); - case tplFmtRFC3164Date: - MsgLock(pM); - if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } - datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); - } - MsgUnlock(pM); - return(pM->pszTIMESTAMP3164); case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszTIMESTAMP3339 == NULL) { - if((pM->pszTIMESTAMP3339 = malloc(33)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ - } - datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339, 33); + pM->pszTIMESTAMP3339 = pM->pszTimestamp3339; + datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339); } MsgUnlock(pM); return(pM->pszTIMESTAMP3339); case tplFmtSecFrac: - MsgLock(pM); - if(pM->pszTIMESTAMP_SecFrac == NULL) { - if((pM->pszTIMESTAMP_SecFrac = malloc(10)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ + if(pM->pszTIMESTAMP_SecFrac[0] == '\0') { + MsgLock(pM); + /* re-check, may have changed while we did not hold lock */ + if(pM->pszTIMESTAMP_SecFrac[0] == '\0') { + datetime.formatTimestampSecFrac(&pM->tTIMESTAMP, pM->pszTIMESTAMP_SecFrac); } - datetime.formatTimestampSecFrac(&pM->tTIMESTAMP, pM->pszTIMESTAMP_SecFrac, 10); + MsgUnlock(pM); } - MsgUnlock(pM); return(pM->pszTIMESTAMP_SecFrac); } ENDfunc return "INVALID eFmt OPTION!"; } -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -848,68 +1334,67 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtDefault: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164, 16); + datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164); } MsgUnlock(pM); return(pM->pszRcvdAt3164); case tplFmtMySQLDate: MsgLock(pM); if(pM->pszRcvdAt_MySQL == NULL) { - if((pM->pszRcvdAt_MySQL = malloc(15)) == NULL) { + if((pM->pszRcvdAt_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToMySQL(&pM->tRcvdAt, pM->pszRcvdAt_MySQL, 15); + datetime.formatTimestampToMySQL(&pM->tRcvdAt, pM->pszRcvdAt_MySQL); } MsgUnlock(pM); return(pM->pszRcvdAt_MySQL); case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszRcvdAt_PgSQL == NULL) { - if((pM->pszRcvdAt_PgSQL = malloc(21)) == NULL) { + if((pM->pszRcvdAt_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestampToPgSQL(&pM->tRcvdAt, pM->pszRcvdAt_PgSQL, 21); + datetime.formatTimestampToPgSQL(&pM->tRcvdAt, pM->pszRcvdAt_PgSQL); } MsgUnlock(pM); return(pM->pszRcvdAt_PgSQL); case tplFmtRFC3164Date: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164, 16); + datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164); } MsgUnlock(pM); return(pM->pszRcvdAt3164); case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszRcvdAt3339 == NULL) { - if((pM->pszRcvdAt3339 = malloc(33)) == NULL) { + if((pM->pszRcvdAt3339 = MALLOC(33)) == NULL) { MsgUnlock(pM); return ""; } - datetime.formatTimestamp3339(&pM->tRcvdAt, pM->pszRcvdAt3339, 33); + datetime.formatTimestamp3339(&pM->tRcvdAt, pM->pszRcvdAt3339); } MsgUnlock(pM); return(pM->pszRcvdAt3339); case tplFmtSecFrac: - MsgLock(pM); - if(pM->pszRcvdAt_SecFrac == NULL) { - if((pM->pszRcvdAt_SecFrac = malloc(10)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ + if(pM->pszRcvdAt_SecFrac[0] == '\0') { + MsgLock(pM); + /* re-check, may have changed while we did not hold lock */ + if(pM->pszRcvdAt_SecFrac[0] == '\0') { + datetime.formatTimestampSecFrac(&pM->tRcvdAt, pM->pszRcvdAt_SecFrac); } - datetime.formatTimestampSecFrac(&pM->tRcvdAt, pM->pszRcvdAt_SecFrac, 10); + MsgUnlock(pM); } - MsgUnlock(pM); return(pM->pszRcvdAt_SecFrac); } ENDfunc @@ -917,103 +1402,69 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) } -char *getSeverity(msg_t *pM) +static inline char *getSeverity(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverity == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverity = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverity = - snprintf((char*)pM->pszSeverity, 2, "%d", pM->iSeverity); + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_number_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverity); + + return name; } -char *getSeverityStr(msg_t *pM) +static inline char *getSeverityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverityStr == NULL) { - for(c = rs_prioritynames, val = pM->iSeverity; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverityStr = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = - snprintf((char*)pM->pszSeverityStr, 2, "%d", pM->iSeverity); - } else { - if((pM->pszSeverityStr = (uchar*) strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = strlen((char*)name); - } + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_severity_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverityStr); + + return name; } -char *getFacility(msg_t *pM) +static inline char *getFacility(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacility == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacility = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacility = - snprintf((char*)pM->pszFacility, 12, "%d", pM->iFacility); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_number_names[pM->iFacility]; } - MsgUnlock(pM); - return((char*)pM->pszFacility); + + return name; } -char *getFacilityStr(msg_t *pM) +static inline char *getFacilityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacilityStr == NULL) { - for(c = rs_facilitynames, val = pM->iFacility << 3; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacilityStr = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = - snprintf((char*)pM->pszFacilityStr, 12, "%d", val >> 3); - } else { - if((pM->pszFacilityStr = (uchar*)strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = strlen((char*)name); - } - } - MsgUnlock(pM); - return((char*)pM->pszFacilityStr); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_fac_names[pM->iFacility]; + } + + return name; } @@ -1035,9 +1486,23 @@ MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl) RETiRet; } +/* set offset after which PRI in raw msg starts + * rgerhards, 2009-06-16 + */ +rsRetVal +MsgSetAfterPRIOffs(msg_t *pMsg, short offs) +{ + assert(pMsg != NULL); + pMsg->offAfterPRI = offs; + return RS_RET_OK; +} + /* rgerhards 2004-11-24: set APP-NAME in msg object - * TODO: revisit msg locking code! + * This is not locked, because it either is called during message + * construction (where we need no locking) or later as part of a function + * which already obtained the lock. So in general, this function here must + * only be called when it it safe to do so without it aquiring a lock. */ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME) { @@ -1046,7 +1511,6 @@ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME) if(pMsg->pCSAPPNAME == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSAPPNAME)); - rsCStrSetAllocIncrement(pMsg->pCSAPPNAME, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSAPPNAME, (uchar*) pszAPPNAME); @@ -1056,20 +1520,6 @@ finalize_it: } -static void tryEmulateAPPNAME(msg_t *pM); /* forward reference */ -/* rgerhards, 2005-11-24 - */ -char *getAPPNAME(msg_t *pM) -{ - assert(pM != NULL); - MsgLock(pM); - if(pM->pCSAPPNAME == NULL) - tryEmulateAPPNAME(pM); - MsgUnlock(pM); - return (pM->pCSAPPNAME == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSAPPNAME); -} - - /* rgerhards 2004-11-24: set PROCID in msg object */ rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID) @@ -1078,42 +1528,54 @@ rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID) ISOBJ_TYPE_assert(pMsg, msg); if(pMsg->pCSPROCID == NULL) { /* we need to obtain the object first */ - CHKiRet(rsCStrConstruct(&pMsg->pCSPROCID)); - rsCStrSetAllocIncrement(pMsg->pCSPROCID, 128); + CHKiRet(cstrConstruct(&pMsg->pCSPROCID)); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSPROCID, (uchar*) pszPROCID); + CHKiRet(cstrFinalize(pMsg->pCSPROCID)); finalize_it: RETiRet; } + +/* check if we have a procid, and, if not, try to aquire/emulate it. + * This must be called WITHOUT the message lock being held. + * rgerhards, 2009-06-26 + */ +static inline void preparePROCID(msg_t *pM, sbool bLockMutex) +{ + if(pM->pCSPROCID == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + /* re-query, things may have changed in the mean time... */ + if(pM->pCSPROCID == NULL) + aquirePROCIDFromTAG(pM); + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } +} + + +#if 0 /* rgerhards, 2005-11-24 */ -int getPROCIDLen(msg_t *pM) +static inline int getPROCIDLen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); - MsgLock(pM); - if(pM->pCSPROCID == NULL) - aquirePROCIDFromTAG(pM); - MsgUnlock(pM); + preparePROCID(pM, bLockMutex); return (pM->pCSPROCID == NULL) ? 1 : rsCStrLen(pM->pCSPROCID); } +#endif /* rgerhards, 2005-11-24 */ -char *getPROCID(msg_t *pM) +char *getPROCID(msg_t *pM, sbool bLockMutex) { - char* pszRet; - ISOBJ_TYPE_assert(pM, msg); - MsgLock(pM); - if(pM->pCSPROCID == NULL) - aquirePROCIDFromTAG(pM); - pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSPROCID); - MsgUnlock(pM); - return pszRet; + preparePROCID(pM, bLockMutex); + return (pM->pCSPROCID == NULL) ? "-" : (char*) cstrGetSzStrNoNULL(pM->pCSPROCID); } @@ -1126,7 +1588,6 @@ rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID) if(pMsg->pCSMSGID == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSMSGID)); - rsCStrSetAllocIncrement(pMsg->pCSMSGID, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSMSGID, (uchar*) pszMSGID); @@ -1135,49 +1596,50 @@ finalize_it: RETiRet; } -/* rgerhards, 2005-11-24 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getMSGIDLen(msg_t *pM) -{ - return (pM->pCSMSGID == NULL) ? 1 : rsCStrLen(pM->pCSMSGID); -} -#endif - /* rgerhards, 2005-11-24 */ -char *getMSGID(msg_t *pM) +static inline char *getMSGID(msg_t *pM) { return (pM->pCSMSGID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSMSGID); } -/* Set the TAG to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetTAG(). - * rgerhards 2004-11-19 +/* rgerhards 2009-06-12: set associated ruleset */ -void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) +void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) { assert(pMsg != NULL); - pMsg->iLenTAG = (pBuf == NULL) ? 0 : strlen((char*)pBuf); - pMsg->pszTAG = (uchar*) pBuf; + pMsg->pRuleset = pRuleset; } -/* rgerhards 2004-11-16: set TAG in msg object +/* set TAG in msg object + * (rewritten 2009-06-18 rgerhards) */ -void MsgSetTAG(msg_t *pMsg, char* pszTAG) +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) { + uchar *pBuf; assert(pMsg != NULL); - if(pMsg->pszTAG != NULL) - free(pMsg->pszTAG); - pMsg->iLenTAG = strlen(pszTAG); - if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) - memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); - else - dbgprintf("Could not allocate memory in MsgSetTAG()\n"); + + freeTAG(pMsg); + + pMsg->iLenTAG = lenBuf; + if(pMsg->iLenTAG < CONF_TAG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pBuf = pMsg->TAG.szBuf; + } else { + if((pBuf = (uchar*) MALLOC(pMsg->iLenTAG + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pBuf = pMsg->TAG.szBuf; + pMsg->iLenTAG = CONF_TAG_BUFSIZE - 1; + } else { + pMsg->TAG.pszTAG = pBuf; + } + } + + memcpy(pBuf, pszBuf, pMsg->iLenTAG); + pBuf[pMsg->iLenTAG] = '\0'; /* this also works with truncation! */ } @@ -1188,63 +1650,51 @@ void MsgSetTAG(msg_t *pMsg, char* pszTAG) * if there is a TAG and, if not, if it can emulate it. * rgerhards, 2005-11-24 */ -static void tryEmulateTAG(msg_t *pM) +static inline void tryEmulateTAG(msg_t *pM, sbool bLockMutex) { - int iTAGLen; - uchar *pBuf; + size_t lenTAG; + uchar bufTAG[CONF_TAG_MAXSIZE]; assert(pM != NULL); - if(pM->pszTAG != NULL) + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + if(pM->iLenTAG > 0) return; /* done, no need to emulate */ if(getProtocolVersion(pM) == 1) { - if(!strcmp(getPROCID(pM), "-")) { + if(!strcmp(getPROCID(pM, MUTEX_ALREADY_LOCKED), "-")) { /* no process ID, use APP-NAME only */ - MsgSetTAG(pM, getAPPNAME(pM)); + MsgSetTAG(pM, (uchar*) getAPPNAME(pM, MUTEX_ALREADY_LOCKED), getAPPNAMELen(pM, MUTEX_ALREADY_LOCKED)); } else { /* now we can try to emulate */ - iTAGLen = getAPPNAMELen(pM) + getPROCIDLen(pM) + 3; - if((pBuf = malloc(iTAGLen * sizeof(char))) == NULL) - return; /* nothing we can do */ - snprintf((char*)pBuf, iTAGLen, "%s[%s]", getAPPNAME(pM), getPROCID(pM)); - MsgAssignTAG(pM, pBuf); + lenTAG = snprintf((char*)bufTAG, CONF_TAG_MAXSIZE, "%s[%s]", + getAPPNAME(pM, MUTEX_ALREADY_LOCKED), getPROCID(pM, MUTEX_ALREADY_LOCKED)); + bufTAG[32] = '\0'; /* just to make sure... */ + MsgSetTAG(pM, bufTAG, lenTAG); } } + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); } -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getTAGLen(msg_t *pM) -{ - if(pM == NULL) - return 0; - else { - tryEmulateTAG(pM); - if(pM->pszTAG == NULL) - return 0; - else - return pM->iLenTAG; - } -} -#endif - - -char *getTAG(msg_t *pM) +static inline void +getTAG(msg_t *pM, uchar **ppBuf, int *piLen) { - char *ret; - - if(pM == NULL) - ret = ""; - else { - MsgLock(pM); - tryEmulateTAG(pM); - if(pM->pszTAG == NULL) - ret = ""; - else - ret = (char*) pM->pszTAG; - MsgUnlock(pM); + if(pM == NULL) { + *ppBuf = UCHAR_CONSTANT(""); + *piLen = 0; + } else { + if(pM->iLenTAG == 0) + tryEmulateTAG(pM, LOCK_MUTEX); + if(pM->iLenTAG == 0) { + *ppBuf = UCHAR_CONSTANT(""); + *piLen = 0; + } else { + *ppBuf = (pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG; + *piLen = pM->iLenTAG; + } } - return(ret); } @@ -1253,9 +1703,13 @@ int getHOSTNAMELen(msg_t *pM) if(pM == NULL) return 0; else - if(pM->pszHOSTNAME == NULL) - return 0; - else + if(pM->pszHOSTNAME == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) + return 0; + else + return prop.GetStringLen(pM->rcvFrom.pRcvFrom); + } else return pM->iLenHOSTNAME; } @@ -1265,48 +1719,42 @@ char *getHOSTNAME(msg_t *pM) if(pM == NULL) return ""; else - if(pM->pszHOSTNAME == NULL) - return ""; - else + if(pM->pszHOSTNAME == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) { + return ""; + } else { + uchar *psz; + int len; + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); + return (char*) psz; + } + } else { return (char*) pM->pszHOSTNAME; -} - - -uchar *getInputName(msg_t *pM) -{ - if(pM == NULL) - return (uchar*) ""; - else - if(pM->pszInputName == NULL) - return (uchar*) ""; - else - return pM->pszInputName; + } } uchar *getRcvFrom(msg_t *pM) { - if(pM == NULL) - return UCHAR_CONSTANT(""); - else - if(pM->pszRcvFrom == NULL) - return UCHAR_CONSTANT(""); - else - return pM->pszRcvFrom; -} - + uchar *psz; + int len; + BEGINfunc -uchar *getRcvFromIP(msg_t *pM) -{ - if(pM == NULL) - return (uchar*) ""; - else - if(pM->pszRcvFromIP == NULL) - return (uchar*) ""; + if(pM == NULL) { + psz = UCHAR_CONSTANT(""); + } else { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) + psz = UCHAR_CONSTANT(""); else - return pM->pszRcvFromIP; + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); + } + ENDfunc + return psz; } + /* rgerhards 2004-11-24: set STRUCTURED DATA in msg object */ rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData) @@ -1316,7 +1764,6 @@ rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData) if(pMsg->pCSStrucData == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSStrucData)); - rsCStrSetAllocIncrement(pMsg->pCSStrucData, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSStrucData, (uchar*) pszStrucData); @@ -1339,107 +1786,56 @@ static int getStructuredDataLen(msg_t *pM) /* get the "STRUCTURED-DATA" as sz string * rgerhards, 2005-11-24 */ -char *getStructuredData(msg_t *pM) +static inline char *getStructuredData(msg_t *pM) { return (pM->pCSStrucData == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSStrucData); } - -/* get the length of the "programname" sz string - * rgerhards, 2005-10-19 +/* check if we have a ProgramName, and, if not, try to aquire/emulate it. + * rgerhards, 2009-06-26 */ -int getProgramNameLen(msg_t *pM) +static inline void prepareProgramName(msg_t *pM, sbool bLockMutex) { - int iRet; + if(pM->pCSProgName == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); - assert(pM != NULL); - MsgLock(pM); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramNameLen()\n", iRet); - MsgUnlock(pM); - return 0; /* best we can do (consistent wiht what getProgramName() returns) */ - } - MsgUnlock(pM); + /* re-query as things might have changed during locking */ + if(pM->pCSProgName == NULL) + aquireProgramName(pM); - return (pM->pCSProgName == NULL) ? 0 : rsCStrLen(pM->pCSProgName); + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } } -/* get the "programname" as sz string +/* get the length of the "programname" sz string * rgerhards, 2005-10-19 */ -char *getProgramName(msg_t *pM) /* this is the non-locking version for internal use */ +int getProgramNameLen(msg_t *pM, sbool bLockMutex) { - int iRet; - char *pszRet; - assert(pM != NULL); - MsgLock(pM); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramName()\n", iRet); - pszRet = ""; /* best we can do */ - } else { - pszRet = (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); - } - - MsgUnlock(pM); - return pszRet; + prepareProgramName(pM, bLockMutex); + return (pM->pCSProgName == NULL) ? 0 : rsCStrLen(pM->pCSProgName); } -/* The code below was an approach without PTHREAD_MUTEX_RECURSIVE - * However, it turned out to be quite complex. So far, we use recursive - * locking, which is OK from a performance point of view, especially as - * we do not anticipate that multithreading msg objects is used often. - * However, we may re-think about using non-recursive locking and I leave this - * code in here to conserve the idea. -- rgerhards, 2008-01-05 - */ -#if 0 -static char *getProgramNameNoLock(msg_t *pM) /* this is the non-locking version for internal use */ -{ - int iRet; - assert(pM != NULL); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramName()\n", iRet); - return ""; /* best we can do */ - } - return (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); -} -char *getProgramName(msg_t *pM) /* this is the external callable version */ +/* get the "programname" as sz string + * rgerhards, 2005-10-19 + */ +uchar *getProgramName(msg_t *pM, sbool bLockMutex) { - char *pszRet; - - MsgLock(pM); - pszRet = getProgramNameNoLock(pM); - MsgUnlock(pM); - return pszRet; + prepareProgramName(pM, bLockMutex); + return (pM->pCSProgName == NULL) ? UCHAR_CONSTANT("") : rsCStrGetSzStrNoNULL(pM->pCSProgName); } -/* an alternative approach has been: */ -/* The macro below is used to generate external function definitions - * for such functions that may also be called internally (and thus have - * both a locking and non-locking implementation. Over time, we could - * reconsider how we handle that. -- rgerhards, 2008-01-05 - */ -#define EXT_LOCKED_FUNC(fName, ret) \ -ret fName(msg_t *pM) \ -{ \ - ret valRet; \ - MsgLock(pM); \ - valRet = fName##NoLock(pM); \ - MsgUnlock(pM); \ - return(valRet); \ -} -EXT_LOCKED_FUNC(getProgramName, char*) -/* in this approach, the external function is provided by the macro and - * needs not to be writen. - */ -#endif /* #if 0 -- saved code */ /* This function tries to emulate APPNAME if it is not present. Its * main use is when we have received a log record via legacy syslog and * now would like to send out the same one via syslog-protocol. + * MUST be called with the Msg Lock locked! */ static void tryEmulateAPPNAME(msg_t *pM) { @@ -1449,80 +1845,164 @@ static void tryEmulateAPPNAME(msg_t *pM) if(getProtocolVersion(pM) == 0) { /* only then it makes sense to emulate */ - MsgSetAPPNAME(pM, getProgramName(pM)); + MsgSetAPPNAME(pM, (char*)getProgramName(pM, MUTEX_ALREADY_LOCKED)); } } + +/* check if we have a APPNAME, and, if not, try to aquire/emulate it. + * This must be called WITHOUT the message lock being held. + * rgerhards, 2009-06-26 + */ +static inline void prepareAPPNAME(msg_t *pM, sbool bLockMutex) +{ + if(pM->pCSAPPNAME == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + + /* re-query as things might have changed during locking */ + if(pM->pCSAPPNAME == NULL) + tryEmulateAPPNAME(pM); + + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } +} + /* rgerhards, 2005-11-24 */ -static int getAPPNAMELen(msg_t *pM) +char *getAPPNAME(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); - if(pM->pCSAPPNAME == NULL) - tryEmulateAPPNAME(pM); - return (pM->pCSAPPNAME == NULL) ? 0 : rsCStrLen(pM->pCSAPPNAME); + prepareAPPNAME(pM, bLockMutex); + return (pM->pCSAPPNAME == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSAPPNAME); } -/* rgerhards 2008-09-10: set pszInputName in msg object +/* rgerhards, 2005-11-24 */ -void MsgSetInputName(msg_t *pMsg, uchar* pszInputName) +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex) { - assert(pMsg != NULL); - if(pMsg->pszInputName != NULL) - free(pMsg->pszInputName); - - pMsg->iLenInputName = ustrlen(pszInputName); - if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { - memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); - } + assert(pM != NULL); + prepareAPPNAME(pM, bLockMutex); + return (pM->pCSAPPNAME == NULL) ? 0 : rsCStrLen(pM->pCSAPPNAME); } -/* rgerhards 2004-11-16: set pszRcvFrom in msg object +/* rgerhards 2008-09-10: set pszInputName in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-16 */ -void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom) +void MsgSetInputName(msg_t *pThis, prop_t *inputName) { - assert(pMsg != NULL); - free(pMsg->pszRcvFrom); + assert(pThis != NULL); - pMsg->iLenRcvFrom = ustrlen(pszRcvFrom); - if((pMsg->pszRcvFrom = malloc(pMsg->iLenRcvFrom + 1)) != NULL) { - memcpy(pMsg->pszRcvFrom, pszRcvFrom, pMsg->iLenRcvFrom + 1); - } + prop.AddRef(inputName); + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); + pThis->pInputName = inputName; } -/* rgerhards 2005-05-16: set pszRcvFromIP in msg object */ +/* Set the pfrominet socket store, so that we can obtain the peer at some + * later time. Note that we do not check if pRcvFrom is already set, so this + * function must only be called during message creation. + * NOTE: msgFlags is NOT set. While this is somewhat a violation of layers, + * it is done because it gains us some performance. So the caller must make + * sure the message flags are properly maintained. For all current callers, + * this is always the case and without extra effort required. + * rgerhards, 2009-11-17 + */ rsRetVal -MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP) -{ +msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa){ DEFiRet; - assert(pMsg != NULL); - if(pMsg->pszRcvFromIP != NULL) { - free(pMsg->pszRcvFromIP); - pMsg->iLenRcvFromIP = 0; - } + assert(pThis->rcvFrom.pRcvFrom == NULL); + + CHKmalloc(pThis->rcvFrom.pfrominet = malloc(sizeof(struct sockaddr_storage))); + memcpy(pThis->rcvFrom.pfrominet, sa, sizeof(struct sockaddr_storage)); - CHKmalloc(pMsg->pszRcvFromIP = (uchar*)strdup((char*)pszRcvFromIP)); - pMsg->iLenRcvFromIP = strlen((char*)pszRcvFromIP); finalize_it: RETiRet; } -/* Set the HOSTNAME to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetHOSTNAME(). - * rgerhards 2004-11-19 +/* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-30 */ -void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) +void MsgSetRcvFrom(msg_t *pThis, prop_t *new) { - assert(pMsg != NULL); - assert(pBuf != NULL); - if(pMsg->pszHOSTNAME != NULL) - free(pMsg->pszHOSTNAME); - pMsg->iLenHOSTNAME = strlen(pBuf); - pMsg->pszHOSTNAME = (uchar*) pBuf; + assert(pThis != NULL); + + prop.AddRef(new); + if(pThis->msgFlags & NEEDS_DNSRESOL) { + if(pThis->rcvFrom.pfrominet != NULL) + free(pThis->rcvFrom.pfrominet); + pThis->msgFlags &= ~NEEDS_DNSRESOL; + } else { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } + pThis->rcvFrom.pRcvFrom = new; +} + + +/* This is used to set the property via a string. This function should not be + * called if there is a reliable way for a caller to make sure that the + * same name can be used across multiple messages. However, if it can not + * ensure that, calling this function is the second best thing, because it + * will re-use the previously created property if it contained the same + * name (but it works only for the immediate previous). + * rgerhards, 2009-06-31 + */ +void MsgSetRcvFromStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp) +{ + assert(pThis != NULL); + assert(ppProp != NULL); + + prop.CreateOrReuseStringProp(ppProp, psz, len); + MsgSetRcvFrom(pThis, *ppProp); +} + + +/* set RcvFromIP name in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-30 + */ +rsRetVal MsgSetRcvFromIP(msg_t *pThis, prop_t *new) +{ + assert(pThis != NULL); + + BEGINfunc + prop.AddRef(new); + if(pThis->pRcvFromIP != NULL) + prop.Destruct(&pThis->pRcvFromIP); + pThis->pRcvFromIP = new; + ENDfunc + return RS_RET_OK; +} + + +/* This is used to set the property via a string. This function should not be + * called if there is a reliable way for a caller to make sure that the + * same name can be used across multiple messages. However, if it can not + * ensure that, calling this function is the second best thing, because it + * will re-use the previously created property if it contained the same + * name (but it works only for the immediate previous). + * rgerhards, 2009-06-31 + */ +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp) +{ + DEFiRet; + assert(pThis != NULL); + + CHKiRet(prop.CreateOrReuseStringProp(ppProp, psz, len)); + MsgSetRcvFromIP(pThis, *ppProp); + +finalize_it: + RETiRet; } @@ -1536,83 +2016,118 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) * we need it. The rest of the code already knows how to handle an * unset HOSTNAME. */ -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) +void MsgSetHOSTNAME(msg_t *pThis, uchar* pszHOSTNAME, int lenHOSTNAME) { - assert(pMsg != NULL); - free(pMsg->pszHOSTNAME); + assert(pThis != NULL); - pMsg->iLenHOSTNAME = ustrlen(pszHOSTNAME); - if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) - memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); - else - DBGPRINTF("Could not allocate memory in MsgSetHOSTNAME()\n"); + freeHOSTNAME(pThis); + + pThis->iLenHOSTNAME = lenHOSTNAME; + if(pThis->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pThis->pszHOSTNAME = pThis->szHOSTNAME; + } else if((pThis->pszHOSTNAME = (uchar*) MALLOC(pThis->iLenHOSTNAME + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pThis->pszHOSTNAME = pThis->szHOSTNAME; + pThis->iLenHOSTNAME = CONF_HOSTNAME_BUFSIZE - 1; + } + + memcpy(pThis->pszHOSTNAME, pszHOSTNAME, pThis->iLenHOSTNAME); + pThis->pszHOSTNAME[pThis->iLenHOSTNAME] = '\0'; /* this also works with truncation! */ } -/* Set the UxTradMsg to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetUxTradMsg(). - * rgerhards 2004-11-19 +/* set the offset of the MSG part into the raw msg buffer + * Note that the offset may be higher than the length of the raw message + * (exactly by one). This can happen if we have a message that does not + * contain any MSG part. */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static void MsgAssignUxTradMsg(msg_t *pMsg, char *pBuf) +void MsgSetMSGoffs(msg_t *pMsg, short offs) { - assert(pMsg != NULL); - assert(pBuf != NULL); - pMsg->iLenUxTradMsg = strlen(pBuf); - pMsg->pszUxTradMsg = pBuf; + ISOBJ_TYPE_assert(pMsg, msg); + pMsg->offMSG = offs; + if(offs > pMsg->iLenRawMsg) { + assert(offs - 1 == pMsg->iLenRawMsg); + pMsg->iLenMSG = 0; + } else { + pMsg->iLenMSG = pMsg->iLenRawMsg - offs; + } } -#endif -/* rgerhards 2004-11-17: set the traditional Unix message in msg object +/* replace the MSG part of a message. The update actually takes place inside + * rawmsg. + * There are two cases: either the new message will be larger than the new msg + * or it will be less than or equal. If it is less than or equal, we can utilize + * the previous message buffer. If it is larger, we can utilize the msg_t-included + * message buffer if it fits in there. If this is not the case, we need to alloc + * a new, larger, chunk and copy over the data to it. Note that this function is + * (hopefully) relatively seldom being called, so some performance impact is + * uncritical. In any case, pszMSG is copied, so if it was dynamically allocated, + * the caller is responsible for freeing it. + * rgerhards, 2009-06-23 */ -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg) +rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG) { - assert(pMsg != NULL); - assert(pszUxTradMsg != NULL); - pMsg->iLenUxTradMsg = strlen(pszUxTradMsg); - if(pMsg->pszUxTradMsg != NULL) - free(pMsg->pszUxTradMsg); - if((pMsg->pszUxTradMsg = malloc(pMsg->iLenUxTradMsg + 1)) != NULL) - memcpy(pMsg->pszUxTradMsg, pszUxTradMsg, pMsg->iLenUxTradMsg + 1); - else - dbgprintf("Could not allocate memory for pszUxTradMsg buffer."); + int lenNew; + uchar *bufNew; + DEFiRet; + ISOBJ_TYPE_assert(pThis, msg); + assert(pszMSG != NULL); + + lenNew = pThis->iLenRawMsg + lenMSG - pThis->iLenMSG; + if(lenMSG > pThis->iLenMSG && lenNew >= CONF_RAWMSG_BUFSIZE) { + /* we have lost our "bet" and need to alloc a new buffer ;) */ + CHKmalloc(bufNew = MALLOC(lenNew + 1)); + memcpy(bufNew, pThis->pszRawMsg, pThis->offMSG); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); + pThis->pszRawMsg = bufNew; + } - return(0); + if(lenMSG > 0) + memcpy(pThis->pszRawMsg + pThis->offMSG, pszMSG, lenMSG); + pThis->pszRawMsg[lenNew] = '\0'; /* this also works with truncation! */ + pThis->iLenRawMsg = lenNew; + pThis->iLenMSG = lenMSG; + +finalize_it: + RETiRet; } -/* rgerhards 2004-11-09: set MSG in msg object +/* set raw message in message object. Size of message is provided. + * rgerhards, 2009-06-16 */ -void MsgSetMSG(msg_t *pMsg, char* pszMSG) +void MsgSetRawMsg(msg_t *pThis, char* pszRawMsg, size_t lenMsg) { - assert(pMsg != NULL); - assert(pszMSG != NULL); + assert(pThis != NULL); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); - if(pMsg->pszMSG != NULL) - free(pMsg->pszMSG); + pThis->iLenRawMsg = lenMsg; + if(pThis->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pThis->pszRawMsg = pThis->szRawMsg; + } else if((pThis->pszRawMsg = (uchar*) MALLOC(pThis->iLenRawMsg + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pThis->pszRawMsg = pThis->szRawMsg; + pThis->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; + } - pMsg->iLenMSG = strlen(pszMSG); - if((pMsg->pszMSG = (uchar*) malloc(pMsg->iLenMSG + 1)) != NULL) - memcpy(pMsg->pszMSG, pszMSG, pMsg->iLenMSG + 1); - else - dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); + memcpy(pThis->pszRawMsg, pszRawMsg, pThis->iLenRawMsg); + pThis->pszRawMsg[pThis->iLenRawMsg] = '\0'; /* this also works with truncation! */ } -/* rgerhards 2004-11-11: set RawMsg in msg object + +/* set raw message in message object. Size of message is not provided. This + * function should only be used when it is unavoidable (and over time we should + * try to remove it altogether). + * rgerhards, 2009-06-16 */ -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg) { - assert(pMsg != NULL); - if(pMsg->pszRawMsg != NULL) - free(pMsg->pszRawMsg); - - pMsg->iLenRawMsg = strlen(pszRawMsg); - if((pMsg->pszRawMsg = (uchar*) malloc(pMsg->iLenRawMsg + 1)) != NULL) - memcpy(pMsg->pszRawMsg, pszRawMsg, pMsg->iLenRawMsg + 1); - else - dbgprintf("Could not allocate memory for pszRawMsg buffer."); + MsgSetRawMsg(pMsg, pszRawMsg, strlen(pszRawMsg)); } @@ -1626,15 +2141,11 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) */ char *textpri(char *pRes, size_t pResLen, int pri) { - syslogCODE *c_pri, *c_fac; - assert(pRes != NULL); assert(pResLen > 0); - for (c_fac = rs_facilitynames; c_fac->c_name && !(c_fac->c_val == LOG_FAC(pri)<<3); c_fac++); - for (c_pri = rs_prioritynames; c_pri->c_name && !(c_pri->c_val == LOG_PRI(pri)); c_pri++); - - snprintf (pRes, pResLen, "%s.%s<%d>", c_fac->c_name, c_pri->c_name, pri); + snprintf(pRes, pResLen, "%s.%s<%d>", syslog_fac_names[LOG_FAC(pri)], + syslog_severity_names[LOG_PRI(pri)], pri); return pRes; } @@ -1654,7 +2165,7 @@ static uchar *getNOW(eNOWType eNow) uchar *pBuf; struct syslogTime t; - if((pBuf = (uchar*) malloc(sizeof(uchar) * tmpBUFSIZE)) == NULL) { + if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { return NULL; } @@ -1728,140 +2239,177 @@ static uchar *getNOW(eNOWType eNow) * be used in selector line processing. * rgerhards 2005-09-15 */ -char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed) -{ - uchar *pName; - char *pRes; /* result pointer */ - char *pBufStart; - char *pBuf; +/* a quick helper to save some writing: */ +#define RET_OUT_OF_MEMORY { *pbMustBeFreed = 0;\ + *pPropLen = sizeof("**OUT OF MEMORY**") - 1; \ + return(UCHAR_CONSTANT("**OUT OF MEMORY**"));} +uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, + propid_t propID, size_t *pPropLen, + unsigned short *pbMustBeFreed) +{ + uchar *pRes; /* result pointer */ + int bufLen = -1; /* length of string or -1, if not known */ + uchar *pBufStart; + uchar *pBuf; int iLen; short iOffs; + BEGINfunc + assert(pMsg != NULL); + assert(pbMustBeFreed != NULL); + #ifdef FEATURE_REGEXP /* Variables necessary for regular expression matching */ size_t nmatch = 10; regmatch_t pmatch[10]; #endif - assert(pMsg != NULL); - assert(pbMustBeFreed != NULL); - - if(pCSPropName == NULL) { - assert(pTpe != NULL); - pName = pTpe->data.field.pPropRepl; - } else { - pName = rsCStrGetSzStrNoNULL(pCSPropName); - } *pbMustBeFreed = 0; - /* sometimes there are aliases to the original MonitoWare - * property names. These come after || in the ifs below. */ - if(!strcmp((char*) pName, "msg")) { - pRes = getMSG(pMsg); - } else if(!strcmp((char*) pName, "rawmsg")) { - pRes = getRawMsg(pMsg); - } else if(!strcmp((char*) pName, "uxtradmsg")) { - pRes = getUxTradMsg(pMsg); - } else if(!strcmp((char*) pName, "inputname")) { - pRes = (char*) getInputName(pMsg); - } else if(!strcmp((char*) pName, "fromhost")) { - pRes = (char*) getRcvFrom(pMsg); - } else if(!strcmp((char*) pName, "fromhost-ip")) { - pRes = (char*) getRcvFromIP(pMsg); - } else if(!strcmp((char*) pName, "source") || !strcmp((char*) pName, "hostname")) { - pRes = getHOSTNAME(pMsg); - } else if(!strcmp((char*) pName, "syslogtag")) { - pRes = getTAG(pMsg); - } else if(!strcmp((char*) pName, "pri")) { - pRes = getPRI(pMsg); - } else if(!strcmp((char*) pName, "pri-text")) { - pBuf = malloc(20 * sizeof(char)); - if(pBuf == NULL) { + switch(propID) { + case PROP_MSG: + pRes = getMSG(pMsg); + bufLen = getMSGLen(pMsg); + break; + case PROP_TIMESTAMP: + pRes = (uchar*)getTimeReported(pMsg, pTpe->data.field.eDateFormat); + break; + case PROP_HOSTNAME: + pRes = (uchar*)getHOSTNAME(pMsg); + bufLen = getHOSTNAMELen(pMsg); + break; + case PROP_SYSLOGTAG: + getTAG(pMsg, &pRes, &bufLen); + break; + case PROP_RAWMSG: + getRawMsg(pMsg, &pRes, &bufLen); + break; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + case PROP_UXTRADMSG: + pRes = getUxTradMsg(pMsg); + break; + */ + case PROP_INPUTNAME: + getInputName(pMsg, &pRes, &bufLen); + break; + case PROP_FROMHOST: + pRes = getRcvFrom(pMsg); + break; + case PROP_FROMHOST_IP: + pRes = getRcvFromIP(pMsg); + break; + case PROP_PRI: + pRes = (uchar*)getPRI(pMsg); + break; + case PROP_PRI_TEXT: + pBuf = MALLOC(20 * sizeof(uchar)); + if(pBuf == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + pRes = (uchar*)textpri((char*)pBuf, 20, getPRIi(pMsg)); + } + break; + case PROP_IUT: + pRes = UCHAR_CONSTANT("1"); /* always 1 for syslog messages (a MonitorWare thing;)) */ + bufLen = 1; + break; + case PROP_SYSLOGFACILITY: + pRes = (uchar*)getFacility(pMsg); + break; + case PROP_SYSLOGFACILITY_TEXT: + pRes = (uchar*)getFacilityStr(pMsg); + break; + case PROP_SYSLOGSEVERITY: + pRes = (uchar*)getSeverity(pMsg); + break; + case PROP_SYSLOGSEVERITY_TEXT: + pRes = (uchar*)getSeverityStr(pMsg); + break; + case PROP_TIMEGENERATED: + pRes = (uchar*)getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); + break; + case PROP_PROGRAMNAME: + pRes = getProgramName(pMsg, LOCK_MUTEX); + break; + case PROP_PROTOCOL_VERSION: + pRes = (uchar*)getProtocolVersionString(pMsg); + break; + case PROP_STRUCTURED_DATA: + pRes = (uchar*)getStructuredData(pMsg); + break; + case PROP_APP_NAME: + pRes = (uchar*)getAPPNAME(pMsg, LOCK_MUTEX); + break; + case PROP_PROCID: + pRes = (uchar*)getPROCID(pMsg, LOCK_MUTEX); + break; + case PROP_MSGID: + pRes = (uchar*)getMSGID(pMsg); + break; + case PROP_SYS_NOW: + if((pRes = getNOW(NOW_NOW)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_YEAR: + if((pRes = getNOW(NOW_YEAR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MONTH: + if((pRes = getNOW(NOW_MONTH)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_DAY: + if((pRes = getNOW(NOW_DAY)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_HOUR: + if((pRes = getNOW(NOW_HOUR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_HHOUR: + if((pRes = getNOW(NOW_HHOUR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_QHOUR: + if((pRes = getNOW(NOW_QHOUR)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MINUTE: + if((pRes = getNOW(NOW_MINUTE)) == NULL) { + RET_OUT_OF_MEMORY; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MYHOSTNAME: + pRes = glbl.GetLocalHostName(); + break; + default: + /* there is no point in continuing, we may even otherwise render the + * error message unreadable. rgerhards, 2007-07-10 + */ + dbgprintf("invalid property id: '%d'\n", propID); *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } else { - *pbMustBeFreed = 1; - pRes = textpri(pBuf, 20, getPRIi(pMsg)); - } - } else if(!strcmp((char*) pName, "iut")) { - pRes = "1"; /* always 1 for syslog messages (a MonitorWare thing;)) */ - } else if(!strcmp((char*) pName, "syslogfacility")) { - pRes = getFacility(pMsg); - } else if(!strcmp((char*) pName, "syslogfacility-text")) { - pRes = getFacilityStr(pMsg); - } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { - pRes = getSeverity(pMsg); - } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { - pRes = getSeverityStr(pMsg); - } else if(!strcmp((char*) pName, "timegenerated")) { - pRes = getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "timereported") - || !strcmp((char*) pName, "timestamp")) { - pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "programname")) { - pRes = getProgramName(pMsg); - } else if(!strcmp((char*) pName, "protocol-version")) { - pRes = getProtocolVersionString(pMsg); - } else if(!strcmp((char*) pName, "structured-data")) { - pRes = getStructuredData(pMsg); - } else if(!strcmp((char*) pName, "app-name")) { - pRes = getAPPNAME(pMsg); - } else if(!strcmp((char*) pName, "procid")) { - pRes = getPROCID(pMsg); - } else if(!strcmp((char*) pName, "msgid")) { - pRes = getMSGID(pMsg); - /* here start system properties (those, that do not relate to the message itself */ - } else if(!strcmp((char*) pName, "$now")) { - if((pRes = (char*) getNOW(NOW_NOW)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$year")) { - if((pRes = (char*) getNOW(NOW_YEAR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$month")) { - if((pRes = (char*) getNOW(NOW_MONTH)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$day")) { - if((pRes = (char*) getNOW(NOW_DAY)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$hour")) { - if((pRes = (char*) getNOW(NOW_HOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$hhour")) { - if((pRes = (char*) getNOW(NOW_HHOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$qhour")) { - if((pRes = (char*) getNOW(NOW_QHOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$minute")) { - if((pRes = (char*) getNOW(NOW_MINUTE)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$myhostname")) { - pRes = (char*) glbl.GetLocalHostName(); - } else { - /* there is no point in continuing, we may even otherwise render the - * error message unreadable. rgerhards, 2007-07-10 - */ - dbgprintf("invalid property name: '%s'\n", pName); - return "**INVALID PROPERTY NAME**"; + *pPropLen = sizeof("**INVALID PROPERTY NAME**") - 1; + return UCHAR_CONSTANT("**INVALID PROPERTY NAME**"); } + /* If we did not receive a template pointer, we are already done... */ if(pTpe == NULL) { return pRes; @@ -1878,8 +2426,8 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.has_fields == 1) { size_t iCurrFld; - char *pFld; - char *pFldEnd; + uchar *pFld; + uchar *pFldEnd; /* first, skip to the field in question. The field separator * is always one character and is stored in the template entry. */ @@ -1912,15 +2460,15 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* we got our end pointer, now do the copy */ /* TODO: code copied from below, this is a candidate for a separate function */ iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } /* now copy */ memcpy(pBuf, pFld, iLen); + bufLen = iLen; pBuf[iLen] = '\0'; /* terminate it */ if(*pbMustBeFreed == 1) free(pRes); @@ -1933,12 +2481,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); *pbMustBeFreed = 0; - return "**FIELD NOT FOUND**"; + *pPropLen = sizeof("**FIELD NOT FOUND**") - 1; + return UCHAR_CONSTANT("**FIELD NOT FOUND**"); } } else if(pTpe->data.field.iFromPos != 0 || pTpe->data.field.iToPos != 0) { /* we need to obtain a private copy */ int iFrom, iTo; - char *pSb; + uchar *pSb; iFrom = pTpe->data.field.iFromPos; iTo = pTpe->data.field.iToPos; /* need to zero-base to and from (they are 1-based!) */ @@ -1946,42 +2495,60 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, --iFrom; if(iTo > 0) --iTo; - iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); - if(pBuf == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } - pSb = pRes; - if(iFrom) { - /* skip to the start of the substring (can't do pointer arithmetic - * because the whole string might be smaller!!) - */ - while(*pSb && iFrom) { - --iFrom; + if(bufLen == -1) + bufLen = ustrlen(pRes); + if(iFrom == 0 && iTo >= bufLen) { + /* in this case, the requested string is a superset of what we already have, + * so there is no need to do any processing. This is a frequent case for size-limited + * fields like TAG in the default forwarding template (so it is a useful optimization + * to check for this condition ;)). -- rgerhards, 2009-07-09 + */ + ; /*DO NOTHING*/ + } else { + iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); + if(pBuf == NULL) { + if(*pbMustBeFreed == 1) + free(pRes); + RET_OUT_OF_MEMORY; + } + pSb = pRes; + if(iFrom) { + /* skip to the start of the substring (can't do pointer arithmetic + * because the whole string might be smaller!!) + */ + while(*pSb && iFrom) { + --iFrom; + ++pSb; + } + } + /* OK, we are at the begin - now let's copy... */ + bufLen = iLen; + while(*pSb && iLen) { + *pBuf++ = *pSb; ++pSb; + --iLen; } + *pBuf = '\0'; + bufLen -= iLen; /* subtract remaining length if the string was smaller! */ + if(*pbMustBeFreed == 1) + free(pRes); + pRes = pBufStart; + *pbMustBeFreed = 1; } - /* OK, we are at the begin - now let's copy... */ - while(*pSb && iLen) { - *pBuf++ = *pSb; - ++pSb; - --iLen; - } - *pBuf = '\0'; - if(*pbMustBeFreed == 1) - free(pRes); - pRes = pBufStart; - *pbMustBeFreed = 1; #ifdef FEATURE_REGEXP } else { /* Check for regular expressions */ if (pTpe->data.field.has_regex != 0) { - if (pTpe->data.field.has_regex == 2) + if (pTpe->data.field.has_regex == 2) { /* Could not compile regex before! */ - return "**NO MATCH** **BAD REGULAR EXPRESSION**"; + if (*pbMustBeFreed == 1) { + free(pRes); + *pbMustBeFreed = 0; + } + *pPropLen = sizeof("**NO MATCH** **BAD REGULAR EXPRESSION**") - 1; + return UCHAR_CONSTANT("**NO MATCH** **BAD REGULAR EXPRESSION**"); + } dbgprintf("string to match for regex is: %s\n", pRes); @@ -1994,7 +2561,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ while(!bFound) { int iREstat; - iREstat = regexp.regexec(&pTpe->data.field.re, pRes + iOffs, nmatch, pmatch, 0); + iREstat = regexp.regexec(&pTpe->data.field.re, (char*)(pRes + iOffs), nmatch, pmatch, 0); dbgprintf("regexec return is %d\n", iREstat); if(iREstat == 0) { if(pmatch[0].rm_so == -1) { @@ -2021,12 +2588,16 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) - return "**NO MATCH**"; - else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) - return "0"; - else - return ""; + if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) { + bufLen = sizeof("**NO MATCH**") - 1; + pRes = UCHAR_CONSTANT("**NO MATCH**"); + } else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) { + bufLen = 1; + pRes = UCHAR_CONSTANT("0"); + } else { + bufLen = 0; + pRes = UCHAR_CONSTANT(""); + } } } else { /* Match- but did it match the one we wanted? */ @@ -2037,29 +2608,35 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) - return "**NO MATCH**"; - else - return ""; + if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) { + bufLen = sizeof("**NO MATCH**") - 1; + pRes = UCHAR_CONSTANT("**NO MATCH**"); + } else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) { + bufLen = 1; + pRes = UCHAR_CONSTANT("0"); + } else { + bufLen = 0; + pRes = UCHAR_CONSTANT(""); + } } } /* OK, we have a usable match - we now need to malloc pB */ int iLenBuf; - char *pB; + uchar *pB; iLenBuf = pmatch[pTpe->data.field.iSubMatchToUse].rm_eo - pmatch[pTpe->data.field.iSubMatchToUse].rm_so; - pB = (char *) malloc((iLenBuf + 1) * sizeof(char)); + pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if (pB == NULL) { if (*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY ALLOCATING pBuf**"; + RET_OUT_OF_MEMORY; } /* Lets copy the matched substring to the buffer */ memcpy(pB, pRes + iOffs + pmatch[pTpe->data.field.iSubMatchToUse].rm_so, iLenBuf); + bufLen = iLenBuf; pB[iLenBuf] = '\0';/* terminate string, did not happen before */ if (*pbMustBeFreed == 1) @@ -2077,7 +2654,8 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - return "***REGEXP NOT AVAILABLE***"; + *pPropLen = sizeof("***REGEXP NOT AVAILABLE***") - 1; + return UCHAR_CONSTANT("***REGEXP NOT AVAILABLE***"); } } #endif /* #ifdef FEATURE_REGEXP */ @@ -2085,28 +2663,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* now check if we need to do our "SP if first char is non-space" hack logic */ if(*pRes && pTpe->data.field.options.bSPIffNo1stSP) { - char *pB; - uchar cFirst = *pRes; - /* here, we always destruct the buffer and return a new one */ - pB = (char *) malloc(2 * sizeof(char)); - if(pB == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } - pRes = pB; - *pbMustBeFreed = 1; - - if(cFirst == ' ') { - /* if we have a SP, we must return an empty string */ - *pRes = '\0'; /* empty */ - } else { - /* if it is no SP, we need to return one */ - *pRes = ' '; - *(pRes+1) = '\0'; - } + uchar cFirst = *pRes; /* save first char */ + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (cFirst == ' ') ? UCHAR_CONSTANT("") : UCHAR_CONSTANT(" "); + bufLen = (cFirst == ' ') ? 0 : 1; + *pbMustBeFreed = 0; } if(*pRes) { @@ -2115,21 +2678,21 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.eCaseConv != tplCaseConvNo) { /* we need to obtain a private copy */ - int iBufLen = strlen(pRes); - char *pBStart; - char *pB; - char *pSrc; - pBStart = pB = malloc((iBufLen + 1) * sizeof(char)); + if(bufLen == -1) + bufLen = ustrlen(pRes); + uchar *pBStart; + uchar *pB; + uchar *pSrc; + pBStart = pB = MALLOC((bufLen + 1) * sizeof(char)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } pSrc = pRes; while(*pSrc) { *pB++ = (pTpe->data.field.eCaseConv == tplCaseConvUpper) ? - (char)toupper((int)*pSrc) : (char)tolower((int)*pSrc); + (uchar)toupper((int)*pSrc) : (uchar)tolower((int)*pSrc); /* currently only these two exist */ ++pSrc; } @@ -2153,10 +2716,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.options.bDropCC) { int iLenBuf = 0; - char *pSrc = pRes; - char *pDstStart; - char *pDst; - char bDropped = 0; + uchar *pSrc = pRes; + uchar *pDstStart; + uchar *pDst; + uchar bDropped = 0; while(*pSrc) { if(!iscntrl((int) *pSrc++)) @@ -2166,12 +2729,11 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(!iscntrl((int) *pSrc)) @@ -2181,12 +2743,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = iLenBuf; *pbMustBeFreed = 1; } } else if(pTpe->data.field.options.bSpaceCC) { - char *pSrc; - char *pDstStart; - char *pDst; + uchar *pSrc; + uchar *pDstStart; + uchar *pDst; if(*pbMustBeFreed == 1) { /* in this case, we already work on dynamic @@ -2199,12 +2762,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst = ' '; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = ustrlen(pRes); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(iscntrl((int) *pSrc)) @@ -2224,7 +2788,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ int iNumCC = 0; int iLenBuf = 0; - char *pB; + uchar *pB; for(pB = pRes ; *pB ; ++pB) { ++iLenBuf; @@ -2234,21 +2798,20 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(iNumCC > 0) { /* if 0, there is nothing to escape, so we are done */ /* OK, let's do the escaping... */ - char *pBStart; - char szCCEsc[8]; /* buffer for escape sequence */ + uchar *pBStart; + uchar szCCEsc[8]; /* buffer for escape sequence */ int i; iLenBuf += iNumCC * 4; - pBStart = pB = malloc((iLenBuf + 1) * sizeof(char)); + pBStart = pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } while(*pRes) { if(iscntrl((int) *pRes)) { - snprintf(szCCEsc, sizeof(szCCEsc), "#%3.3d", *pRes); + snprintf((char*)szCCEsc, sizeof(szCCEsc), "#%3.3d", *pRes); for(i = 0 ; i < 4 ; ++i) *pB++ = szCCEsc[i]; } else { @@ -2260,6 +2823,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } } @@ -2271,10 +2835,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pTpe->data.field.options.bSecPathDrop || pTpe->data.field.options.bSecPathReplace) { if(pTpe->data.field.options.bSecPathDrop) { int iLenBuf = 0; - char *pSrc = pRes; - char *pDstStart; - char *pDst; - char bDropped = 0; + uchar *pSrc = pRes; + uchar *pDstStart; + uchar *pDst; + uchar bDropped = 0; while(*pSrc) { if(*pSrc++ != '/') @@ -2284,12 +2848,11 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(*pSrc != '/') @@ -2299,12 +2862,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = -1; /* TODO: can we do better? */ *pbMustBeFreed = 1; } } else { - char *pSrc; - char *pDstStart; - char *pDst; + uchar *pSrc; + uchar *pDstStart; + uchar *pDst; if(*pbMustBeFreed == 1) { /* here, again, we can modify the string as we already obtained @@ -2317,12 +2881,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst++ = '_'; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = ustrlen(pRes); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(*pSrc == '/') @@ -2342,44 +2907,48 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* check for "." and ".." (note the parenthesis in the if condition!) */ if((*pRes == '.') && (*(pRes + 1) == '\0' || (*(pRes + 1) == '.' && *(pRes + 2) == '\0'))) { - char *pTmp = pRes; + uchar *pTmp = pRes; if(*(pRes + 1) == '\0') - pRes = "_"; + pRes = UCHAR_CONSTANT("_"); else - pRes = "_.";; + pRes = UCHAR_CONSTANT("_.");; if(*pbMustBeFreed == 1) free(pTmp); *pbMustBeFreed = 0; } else if(*pRes == '\0') { if(*pbMustBeFreed == 1) free(pRes); - pRes = "_"; + pRes = UCHAR_CONSTANT("_"); + bufLen = 1; *pbMustBeFreed = 0; } } /* Now drop last LF if present (pls note that this must not be done - * if bEscapeCC was set! + * if bEscapeCC was set)! */ if(pTpe->data.field.options.bDropLastLF && !pTpe->data.field.options.bEscapeCC) { - int iLn = strlen(pRes); - char *pB; + int iLn; + uchar *pB; + if(bufLen == -1) + bufLen = ustrlen(pRes); + iLn = bufLen; if(iLn > 0 && *(pRes + iLn - 1) == '\n') { /* we have a LF! */ /* check if we need to obtain a private copy */ if(*pbMustBeFreed == 0) { /* ok, original copy, need a private one */ - pB = malloc((iLn + 1) * sizeof(char)); + pB = MALLOC((iLn + 1) * sizeof(uchar)); if(pB == NULL) { - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } memcpy(pB, pRes, iLn - 1); pRes = pB; *pbMustBeFreed = 1; } *(pRes + iLn - 1) = '\0'; /* drop LF ;) */ + --bufLen; } } @@ -2390,19 +2959,23 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.options.bCSV) { /* we need to obtain a private copy, as we need to at least add the double quotes */ - int iBufLen = strlen(pRes); - char *pBStart; - char *pDst; - char *pSrc; + int iBufLen; + int i; + uchar *pBStart; + uchar *pDst; + uchar *pSrc; + if(bufLen == -1) + bufLen = ustrlen(pRes); + iBufLen = bufLen; /* the malloc may be optimized, we currently use the worst case... */ - pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(char)); + pBStart = pDst = MALLOC((2 * iBufLen + 3) * sizeof(uchar)); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; + RET_OUT_OF_MEMORY; } pSrc = pRes; + i = 0; *pDst++ = '"'; /* starting quote */ while(*pSrc) { if(*pSrc == '"') @@ -2414,10 +2987,15 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } - /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ + if(bufLen == -1) + bufLen = ustrlen(pRes); + *pPropLen = bufLen; + + ENDfunc return(pRes); } @@ -2432,8 +3010,10 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) { DEFiRet; var_t *pVar; + size_t propLen; uchar *pszProp = NULL; cstr_t *pstrProp; + propid_t propid; unsigned short bMustBeFreed = 0; ISOBJ_TYPE_assert(pThis, msg); @@ -2445,7 +3025,9 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) CHKiRet(var.ConstructFinalize(pVar)); /* always call MsgGetProp() without a template specifier */ - pszProp = (uchar*) MsgGetProp(pThis, NULL, pstrPropName, &bMustBeFreed); + /* TODO: optimize propNameToID() call -- rgerhards, 2009-06-26 */ + propNameToID(pstrPropName, &propid); + pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, &propLen, &bMustBeFreed); /* now create a string object out of it and hand that over to the var */ CHKiRet(rsCStrConstructFromszStr(&pstrProp, pszProp)); @@ -2460,8 +3042,6 @@ finalize_it: RETiRet; } - - /* This function can be used as a generic way to set properties. * We have to handle a lot of legacy, so our return value is not always * 100% correct (called functions do not always provide one, should @@ -2471,6 +3051,9 @@ finalize_it: #define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) { + prop_t *myProp; + prop_t *propRcvFrom = NULL; + prop_t *propRcvFromIP = NULL; DEFiRet; ISOBJ_TYPE_assert(pThis, msg); @@ -2484,22 +3067,34 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) pThis->iFacility = pProp->val.num; } else if(isProp("msgFlags")) { pThis->msgFlags = pProp->val.num; + } else if(isProp("offMSG")) { + MsgSetMSGoffs(pThis, pProp->val.num); } else if(isProp("pszRawMsg")) { - MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); - } else if(isProp("pszMSG")) { - MsgSetMSG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(isProp("offAfterPRI")) { + pThis->offAfterPRI = pProp->val.num; + */ } else if(isProp("pszUxTradMsg")) { - MsgSetUxTradMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { - MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetTAG(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /* we need to create a property */ + CHKiRet(prop.Construct(&myProp)); + CHKiRet(prop.SetString(myProp, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr))); + CHKiRet(prop.ConstructFinalize(myProp)); + MsgSetInputName(pThis, myProp); + prop.Destruct(&myProp); } else if(isProp("pszRcvFromIP")) { - MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFromIPStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr), &propRcvFromIP); + prop.Destruct(&propRcvFromIP); } else if(isProp("pszRcvFrom")) { - MsgSetRcvFrom(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFromStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr), &propRcvFrom); + prop.Destruct(&propRcvFrom); } else if(isProp("pszHOSTNAME")) { - MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pCSStrucData")) { MsgSetStructuredData(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSAPPNAME")) { @@ -2514,8 +3109,11 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) memcpy(&pThis->tRcvdAt, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); } else if(isProp("tTIMESTAMP")) { memcpy(&pThis->tTIMESTAMP, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); + } else if(isProp("pszMSG")) { + dbgprintf("no longer supported property pszMSG silently ignored\n"); } +finalize_it: RETiRet; } #undef isProp @@ -2559,6 +3157,7 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_SERIALIZE, MsgSerialize); diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..b4b6d9f8 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -3,7 +3,7 @@ * * File begun on 2007-07-13 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -33,6 +33,7 @@ #include "syslogd-types.h" #include "template.h" + /* rgerhards 2004-11-08: The following structure represents a * syslog message. * @@ -47,59 +48,53 @@ * will be decremented. If it is 1, however, the object is actually * destroyed. To make this work, it is vital that MsgAddRef() is * called each time a "copy" is stored somewhere. + * + * WARNING: this structure is not calloc()ed, so be careful when + * adding new fields. You need to initialize them in + * msgBaseConstruct(). That function header comment also describes + * why this is the case. */ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - pthread_mutexattr_t mutAttr; -short bDoLock; /* use the mutex? */ - pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ + pthread_mutex_t mut; + sbool bDoLock; /* use the mutex? */ short iRefCount; /* reference counter (0 = unused) */ - short bParseHOSTNAME; /* should the hostname be parsed from the message? */ - /* background: the hostname is not present on "regular" messages - * received via UNIX domain sockets from the same machine. However, - * it is available when we have a forwarder (e.g. rfc3195d) using local - * sockets. All in all, the parser would need parse templates, that would - * resolve all these issues... rgerhards, 2005-10-06 - */ short iSeverity; /* the severity 0..7 */ - uchar *pszSeverity; /* severity as string... */ - int iLenSeverity; /* ... and its length. */ - uchar *pszSeverityStr; /* severity name... */ - int iLenSeverityStr; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ - uchar *pszFacility; /* Facility as string... */ - int iLenFacility; /* ... and its length. */ - uchar *pszFacilityStr; /* facility name... */ - int iLenFacilityStr; /* ... and its length. */ - uchar *pszPRI; /* the PRI as a string */ - int iLenPRI; /* and its length */ - uchar *pszRawMsg; /* message as it was received on the - * wire. This is important in case we - * need to preserve cryptographic verifiers. - */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ + short offMSG; /* offset at which the MSG part starts in pszRawMsg */ + short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + int msgFlags; /* flags associated with this message */ int iLenRawMsg; /* length of raw message */ - uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszUxTradMsg; /* the traditional UNIX message */ - int iLenUxTradMsg;/* Length of the traditional UNIX message */ - uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ - uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ int iLenHOSTNAME; /* Length of HOSTNAME */ - uchar *pszRcvFrom; /* System message was received from */ - int iLenRcvFrom; /* Length of pszRcvFrom */ - uchar *pszRcvFromIP; /* IP of system message was received from */ - int iLenRcvFromIP; /* Length of pszRcvFromIP */ - uchar *pszInputName; /* name of the input module that submitted this message */ - int iLenInputName; /* Length of pszInputName */ - short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we + * need to preserve cryptographic verifiers. */ + uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ + char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ + char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ + char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ + char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ + char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ + char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ + char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ + char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ cstr_t *pCSProgName; /* the (BSD) program name */ cstr_t *pCSStrucData; /* STRUCTURED-DATA */ cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + prop_t *pInputName; /* input name property */ + prop_t *pRcvFromIP; /* IP of system message was received from */ + union { + prop_t *pRcvFrom;/* name of system message was received from */ + struct sockaddr_storage *pfrominet; /* unresolved name */ + } rcvFrom; + + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp is used at later processing stages (namely in the output arena). Thanks to @@ -108,18 +103,18 @@ short bDoLock; /* use the mutex? */ enough to reliable, but I prefer to leave the subtle things to the OS, where it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ - char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ - char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ - char *pszRcvdAt_SecFrac;/* time just as fractional seconds (6 charcters) */ - char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ - char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ - char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ - char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ - char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ - char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ - char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ - int msgFlags; /* flags associated with this message */ + /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ + uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE]; + union { + uchar *pszTAG; /* pointer to tag value */ + uchar szBuf[CONF_TAG_BUFSIZE]; + } TAG; + char pszTimestamp3164[16]; + char pszTimestamp3339[33]; + char pszTIMESTAMP_SecFrac[7]; /* Note: a pointer is 64 bits/8 char, so this is actually fewer than a pointer! */ + char pszRcvdAt_SecFrac[7]; /* same as above. Both are fractional seconds for their respective timestamp */ }; @@ -132,64 +127,61 @@ short bDoLock; /* use the mutex? */ #define MARK 0x008 /* this message is a mark */ #define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ #define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ +#define NEEDS_DNSRESOL 0x040 /* fromhost address is unresolved and must be locked up via DNS reverse lookup first */ +#define NEEDS_ACLCHK_U 0x080 /* check UDP ACLs after DNS resolution has been done in main queue consumer */ /* function prototypes */ PROTOTYPEObjClassInit(msg); -char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -int getProtocolVersion(msg_t *pM); -char *getProtocolVersionString(msg_t *pM); -int getMSGLen(msg_t *pM); -char *getRawMsg(msg_t *pM); -char *getUxTradMsg(msg_t *pM); -char *getMSG(msg_t *pM); -char *getPRI(msg_t *pM); -int getPRIi(msg_t *pM); -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt); -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt); -char *getSeverity(msg_t *pM); -char *getSeverityStr(msg_t *pM); -char *getFacility(msg_t *pM); -char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, uchar*); +void MsgSetInputName(msg_t *pMsg, prop_t*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); -char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); -int getPROCIDLen(msg_t *pM); -char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); -void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); -void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); +void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); -char *getTAG(msg_t *pM); -int getHOSTNAMELen(msg_t *pM); -char *getHOSTNAME(msg_t *pM); -uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -char *getStructuredData(msg_t *pM); -int getProgramNameLen(msg_t *pM); -char *getProgramName(msg_t *pM); -void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); -rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); -void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); -void MsgSetMSG(msg_t *pMsg, char* pszMSG); -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); -void moveHOSTNAMEtoTAG(msg_t *pM); -char *getMSGID(msg_t *pM); -char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed); +rsRetVal msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa); +void MsgSetRcvFrom(msg_t *pMsg, prop_t*); +void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **); +rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp); +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); +rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); +void MsgSetMSGoffs(msg_t *pMsg, short offs); +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); +rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); +uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, + propid_t propID, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +uchar *getRcvFrom(msg_t *pM); + + +/* TODO: remove these five (so far used in action.c) */ +uchar *getMSG(msg_t *pM); +char *getHOSTNAME(msg_t *pM); +char *getPROCID(msg_t *pM, sbool bLockMutex); +char *getAPPNAME(msg_t *pM, sbool bLockMutex); +int getMSGLen(msg_t *pM); + +char *getHOSTNAME(msg_t *pM); +int getHOSTNAMELen(msg_t *pM); +uchar *getProgramName(msg_t *pM, sbool bLockMutex); +int getProgramNameLen(msg_t *pM, sbool bLockMutex); +uchar *getRcvFrom(msg_t *pM); +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); +uchar *propIDToName(propid_t propID); + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains @@ -200,6 +192,33 @@ rsRetVal MsgEnableThreadSafety(void); extern void (*funcMsgPrepareEnqueue)(msg_t *pMsg); #define MsgPrepareEnqueue(pMsg) funcMsgPrepareEnqueue(pMsg) + +/* ------------------------------ some inline functions ------------------------------ */ + +/* set raw message size. This is needed in some cases where a trunctation is necessary + * but the raw message must not be newly set. The most important (and currently only) + * use case is if we remove trailing LF or NUL characters. Note that the size can NOT + * be extended, only shrunk! + * rgerhards, 2009-08-26 + */ +static inline void +MsgSetRawMsgSize(msg_t *pMsg, size_t newLen) +{ + assert(newLen <= (size_t) pMsg->iLenRawMsg); + pMsg->iLenRawMsg = newLen; +} + + +/* get the ruleset that is associated with the ruleset. + * May be NULL. -- rgerhards, 2009-10-27 + */ +static inline ruleset_t* +MsgGetRuleset(msg_t *pMsg) +{ + return pMsg->pRuleset; +} + + #endif /* #ifndef MSG_H_INCLUDED */ /* vim:set ai: */ diff --git a/runtime/net.c b/runtime/net.c index db2d7e37..ab431f7c 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -173,7 +173,7 @@ AddPermittedPeerWildcard(permittedPeers_t *pPeer, uchar* pszStr, size_t lenStr) /* alloc memory for the domain component. We may waste a byte or * two, but that's ok. */ - CHKmalloc(pNew->pszDomainPart = malloc(lenStr +1 )); + CHKmalloc(pNew->pszDomainPart = MALLOC(lenStr +1 )); } if(pszStr[0] == '*') { @@ -695,7 +695,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS case AF_INET: /* add IPv4 */ iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -710,7 +710,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(sizeof(struct sockaddr_in))) + if((allowIP.addr.NetAddr = MALLOC(sizeof(struct sockaddr_in))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -732,7 +732,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 128; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -892,15 +892,18 @@ rsRetVal addAllowedSenderLine(char* pName, uchar** ppRestOfConfLine) * including IPv4/v6 as well as domain name wildcards. * This is a helper to isAllowedSender. As it is only called once, it is * declared inline. - * Returns 0 if they do not match, something else otherwise. - * contributed 1007-07-16 by mildew@gmail.com + * Returns 0 if they do not match, 1 if they match and 2 if a DNS name would have been required. + * contributed 2007-07-16 by mildew@gmail.com */ -static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost) +static inline int +MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { assert(pAllow != NULL); assert(pFrom != NULL); if(F_ISSET(pAllow->flags, ADDR_NAME)) { + if(bChkDNS == 0) + return 2; dbgprintf("MaskCmp: host=\"%s\"; pattern=\"%s\"\n", pszFromHost, pAllow->addr.HostWildcard); # if !defined(FNM_CASEFOLD) @@ -967,18 +970,22 @@ static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr /* check if a sender is allowed. The root of the the allowed sender. * list must be proveded by the caller. As such, this function can be * used to check both UDP and TCP allowed sender lists. - * returns 1, if the sender is allowed, 0 otherwise. + * returns 1, if the sender is allowed, 0 if not and 2 if we could not + * obtain a result because we would need a dns name, which we don't have + * (2 was added rgerhards, 2009-11-16). * rgerhards, 2005-09-26 */ -static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) +static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { struct AllowedSenders *pAllow; struct AllowedSenders *pAllowRoot; + int bNeededDNS = 0; /* partial check because we could not resolve DNS? */ + int ret; assert(pFrom != NULL); if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK) - return 0; /* if something went wrong, we denie access - that's the better choice... */ + return 0; /* if something went wrong, we deny access - that's the better choice... */ if(pAllowRoot == NULL) return 1; /* checking disabled, everything is valid! */ @@ -990,10 +997,20 @@ static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *p * that the sender is disallowed. */ for(pAllow = pAllowRoot ; pAllow != NULL ; pAllow = pAllow->pNext) { - if (MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost)) + ret = MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost, bChkDNS); + if(ret == 1) return 1; + else if(ret == 2) + bNeededDNS = 2; } - return 0; + return bNeededDNS; +} + + +/* legacy API, not to be used any longer */ +static int +isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) { + return isAllowedSender2(pszType, pFrom, pszFromHost, 1); } @@ -1306,7 +1323,7 @@ getLocalHostname(uchar **ppName) do { if(buf == NULL) { buf_len = 128; /* Initial guess */ - CHKmalloc(buf = malloc(buf_len)); + CHKmalloc(buf = MALLOC(buf_len)); } else { buf_len += buf_len; CHKmalloc(buf = realloc (buf, buf_len)); @@ -1370,7 +1387,7 @@ int *create_udp_socket(uchar *hostname, uchar *pszPort, int bIsServer) /* Count max number of sockets we may open */ for (maxs = 0, r = res; r != NULL ; r = r->ai_next, maxs++) /* EMPTY */; - socks = malloc((maxs+1) * sizeof(int)); + socks = MALLOC((maxs+1) * sizeof(int)); if (socks == NULL) { errmsg.LogError(0, NO_ERRCODE, "couldn't allocate memory for UDP sockets, suspending UDP message reception"); freeaddrinfo(res); @@ -1496,6 +1513,73 @@ int *create_udp_socket(uchar *hostname, uchar *pszPort, int bIsServer) } +/* check if two provided socket addresses point to the same host. Note that the + * length of the sockets must be provided as third parameter. This is necessary to + * compare non IPv4/v6 hosts, in which case we do a simple memory compare of the + * address structure (in that case, the same host may not reliably be detected). + * Note that we need to do the comparison not on the full structure, because it contains things + * like the port, which we do not need to look at when thinking about hostnames. So we look + * at the relevant fields, what means a somewhat more complicated processing. + * Also note that we use a non-standard calling interface, as this is much more natural and + * it looks extremely unlikely that we get an exception of any kind here. What we + * return is mimiced after memcmp(), and as such useful for building binary trees + * (the order relation may be a bit arbritrary, but at least it is consistent). + * rgerhards, 2009-09-03 + */ +static int CmpHost(struct sockaddr_storage *s1, struct sockaddr_storage* s2, size_t socklen) +{ + int ret; + + if(((struct sockaddr*) s1)->sa_family != ((struct sockaddr*) s2)->sa_family) { + ret = memcmp(s1, s2, socklen); + goto finalize_it; + } + + if(((struct sockaddr*) s1)->sa_family == AF_INET) { + if(((struct sockaddr_in *) s1)->sin_addr.s_addr == ((struct sockaddr_in*)s2)->sin_addr.s_addr) { + ret = 0; + } else if(((struct sockaddr_in *) s1)->sin_addr.s_addr < ((struct sockaddr_in*)s2)->sin_addr.s_addr) { + ret = -1; + } else { + ret = 1; + } + } else if(((struct sockaddr*) s1)->sa_family == AF_INET6) { + /* IPv6 addresses are always 16 octets long */ + ret = memcmp(((struct sockaddr_in6 *)s1)->sin6_addr.s6_addr, ((struct sockaddr_in6*)s2)->sin6_addr.s6_addr, 16); + } else { + ret = memcmp(s1, s2, socklen); + } + +finalize_it: + return ret; +} + + + +/* check if restrictions (ALCs) exists. The goal of this function is to disable the + * somewhat time-consuming ACL checks if no restrictions are defined (the usual case). + * This also permits to gain some speedup by using firewall-based ACLs instead of + * rsyslog ACLs (the recommended method. + * rgerhards, 2009-11-16 + */ +static rsRetVal +HasRestrictions(uchar *pszType, int *bHasRestrictions) { + struct AllowedSenders *pAllowRoot; + DEFiRet; + + CHKiRet(setAllowRoot(&pAllowRoot, pszType)); + + *bHasRestrictions = (pAllowRoot == NULL) ? 0 : 1; + +finalize_it: + if(iRet != RS_RET_OK) { + *bHasRestrictions = 1; /* in this case it is better to check individually */ + DBGPRINTF("Error %d trying to obtain ACL restriction state of '%s'\n", iRet, pszType); + } + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-03-05 */ @@ -1519,11 +1603,17 @@ CODESTARTobjQueryInterface(net) pIf->create_udp_socket = create_udp_socket; pIf->closeUDPListenSockets = closeUDPListenSockets; pIf->isAllowedSender = isAllowedSender; + pIf->isAllowedSender2 = isAllowedSender2; pIf->should_use_so_bsdcompat = should_use_so_bsdcompat; pIf->getLocalHostname = getLocalHostname; pIf->AddPermittedPeer = AddPermittedPeer; pIf->DestructPermittedPeers = DestructPermittedPeers; pIf->PermittedPeerWildcardMatch = PermittedPeerWildcardMatch; + pIf->CmpHost = CmpHost; + pIf->HasRestrictions = HasRestrictions; + /* data members */ + pIf->pACLAddHostnameOnFail = &ACLAddHostnameOnFail; + pIf->pACLDontResolve = &ACLDontResolve; finalize_it: ENDobjQueryInterface(net) diff --git a/runtime/net.h b/runtime/net.h index 092c3116..101ce79d 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -139,18 +139,23 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ void (*debugListenInfo)(int fd, char *type); int *(*create_udp_socket)(uchar *hostname, uchar *LogPort, int bIsServer); void (*closeUDPListenSockets)(int *finet); - int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); + int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); /* deprecated! */ rsRetVal (*getLocalHostname)(uchar**); int (*should_use_so_bsdcompat)(void); /* permitted peer handling should be replaced by something better (see comments above) */ rsRetVal (*AddPermittedPeer)(permittedPeers_t **ppRootPeer, uchar *pszID); rsRetVal (*DestructPermittedPeers)(permittedPeers_t **ppRootPeer); rsRetVal (*PermittedPeerWildcardMatch)(permittedPeers_t *pPeer, uchar *pszNameToMatch, int *pbIsMatching); + /* v5 interface additions */ + int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t); + /* v6 interface additions - 2009-11-16 */ + rsRetVal (*HasRestrictions)(uchar *, int *bHasRestrictions); + int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS); /* data members - these should go away over time... TODO */ int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ ENDinterface(net) -#define netCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +#define netCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* prototypes */ PROTOTYPEObj(net); diff --git a/runtime/netstrm.c b/runtime/netstrm.c index f6a8de7f..3658006f 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -114,7 +114,6 @@ AcceptConnReq(netstrm_t *pThis, netstrm_t **ppNew) ISOBJ_TYPE_assert(pThis, netstrm); assert(ppNew != NULL); -RUNLOG_STR("XXX: accept conn reqeust"); /* accept the new connection */ CHKiRet(pThis->Drvr.AcceptConnReq(pThis->pDrvrData, &pNewNsd)); /* construct our object so that we can use it... */ @@ -234,6 +233,19 @@ Send(netstrm_t *pThis, uchar *pBuf, ssize_t *pLenBuf) RETiRet; } +/* Enable Keep-Alive handling for those drivers that support it. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(netstrm_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, netstrm); + iRet = pThis->Drvr.EnableKeepAlive(pThis->pDrvrData); + RETiRet; +} + + /* check connection - slim wrapper for NSD driver function */ static void @@ -338,6 +350,7 @@ CODESTARTobjQueryInterface(netstrm) pIf->SetDrvrPermPeers = SetDrvrPermPeers; pIf->CheckConnection = CheckConnection; pIf->GetSock = GetSock; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(netstrm) diff --git a/runtime/netstrm.h b/runtime/netstrm.h index b00dd223..f6931104 100644 --- a/runtime/netstrm.h +++ b/runtime/netstrm.h @@ -69,9 +69,13 @@ BEGINinterface(netstrm) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v4 */ + rsRetVal (*EnableKeepAlive)(netstrm_t *pThis); ENDinterface(netstrm) -#define netstrmCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ -/* interface version 3 added GetRemAddr() */ +#define netstrmCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +/* interface version 3 added GetRemAddr() + * interface version 4 added EnableKeepAlive() -- rgerhards, 2009-06-02 + * */ /* prototypes */ PROTOTYPEObj(netstrm); diff --git a/runtime/netstrms.c b/runtime/netstrms.c index 6b28e7ea..e9ff2568 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -36,6 +36,7 @@ #include "nsd.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "netstrms.h" MODULE_TYPE_LIB @@ -304,6 +305,7 @@ ENDObjClassInit(netstrms) BEGINmodExit CODESTARTmodExit nsselClassExit(); + nspollClassExit(); netstrmsClassExit(); netstrmClassExit(); /* we use this object, so we must exit it after we are finished */ ENDmodExit @@ -322,6 +324,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(netstrmClassInit(pModInfo)); CHKiRet(nsselClassInit(pModInfo)); + CHKiRet(nspollClassInit(pModInfo)); CHKiRet(netstrmsClassInit(pModInfo)); ENDmodInit /* vi:set ai: diff --git a/runtime/nsd.h b/runtime/nsd.h index f0c9b9b6..e5b9320b 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -69,9 +69,13 @@ BEGINinterface(nsd) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v5 */ + rsRetVal (*EnableKeepAlive)(nsd_t *pThis); ENDinterface(nsd) -#define nsdCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ -/* interface version 4 added GetRemAddr() */ +#define nsdCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +/* interface version 4 added GetRemAddr() + * interface version 5 added EnableKeepAlive() -- rgerhards, 2009-06-02 + */ /* interface for the select call */ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ @@ -83,4 +87,13 @@ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ ENDinterface(nsdsel) #define nsdselCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface for the epoll call */ +BEGINinterface(nsdpoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nsdpoll_t **ppThis); + rsRetVal (*Destruct)(nsdpoll_t **ppThis); + rsRetVal (*Ctl)(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op); + rsRetVal (*Wait)(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); +ENDinterface(nsdpoll) +#define nsdpollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + #endif /* #ifndef INCLUDED_NSD_H */ diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 5786e191..0ee70e56 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -44,6 +44,7 @@ #include "stringbuf.h" #include "errmsg.h" #include "net.h" +#include "datetime.h" #include "nsd_ptcp.h" #include "nsdsel_gtls.h" #include "nsd_gtls.h" @@ -61,6 +62,7 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) +DEFobjCurrIf(datetime) DEFobjCurrIf(nsd_ptcp) static int bGlblSrvrInitDone = 0; /**< 0 - server global init not yet done, 1 - already done */ @@ -129,7 +131,7 @@ readFile(uchar *pszFile, gnutls_datum_t *pBuf) ABORT_FINALIZE(RS_RET_FILE_TOO_LARGE); } - CHKmalloc(pBuf->data = malloc(stat_st.st_size)); + CHKmalloc(pBuf->data = MALLOC(stat_st.st_size)); pBuf->size = stat_st.st_size; if(read(fd, pBuf->data, stat_st.st_size) != stat_st.st_size) { errmsg.LogError(0, RS_RET_IO_ERROR, "error or incomplete read of file '%s'", pszFile); @@ -335,7 +337,7 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) gnutls_x509_crt_deinit(cert); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; finalize_it: @@ -455,7 +457,7 @@ GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) snprintf((char*)buf, sizeof(buf), ":%2.2X", pFingerprint[i]); CHKiRet(rsCStrAppendStrWithLen(pStr, buf, 3)); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -710,20 +712,20 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) } /* we found a common name, now extract it */ - CHKiRet(rsCStrConstruct(&pstrCN)); + CHKiRet(cstrConstruct(&pstrCN)); while(szDN[i] != '\0' && szDN[i] != ',') { if(szDN[i] == '\\') { /* hex escapes are not implemented */ ++i; /* escape char processed */ if(szDN[i] == '\0') ABORT_FINALIZE(RS_RET_CERT_INVALID_DN); - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } else { - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } ++i; /* char processed */ } - CHKiRet(rsCStrFinish(pstrCN)); + CHKiRet(cstrFinalize(pstrCN)); /* we got it - we ignore the rest of the DN string (if any). So we may * not detect if it contains more than one CN @@ -734,7 +736,7 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) finalize_it: if(iRet != RS_RET_OK) { if(pstrCN != NULL) - rsCStrDestruct(&pstrCN); + cstrDestruct(&pstrCN); } RETiRet; @@ -761,7 +763,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) size = sizeof(fingerprint); CHKgnutls(gnutls_x509_crt_get_fingerprint(*pCert, GNUTLS_DIG_SHA1, fingerprint, &size)); CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint)); - dbgprintf("peer's certificate SHA1 fingerprint: %s\n", rsCStrGetSzStr(pstrFingerprint)); + dbgprintf("peer's certificate SHA1 fingerprint: %s\n", cstrGetSzStr(pstrFingerprint)); /* now search through the permitted peers to see if we can find a permitted one */ bFoundPositiveMatch = 0; @@ -779,7 +781,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) if(pThis->bReportAuthErr == 1) { errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer fingerprint '%s' unknown - we are " - "not permitted to talk to it", rsCStrGetSzStr(pstrFingerprint)); + "not permitted to talk to it", cstrGetSzStr(pstrFingerprint)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -787,7 +789,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) finalize_it: if(pstrFingerprint != NULL) - rsCStrDestruct(&pstrFingerprint); + cstrDestruct(&pstrFingerprint); RETiRet; } @@ -874,21 +876,21 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) /* if we did not succeed so far, we try the CN part of the DN... */ CHKiRet(gtlsGetCN(pThis, pCert, &pstrCN)); if(pstrCN != NULL) { /* NULL if there was no CN present */ - dbgprintf("gtls now checking auth for CN '%s'\n", rsCStrGetSzStr(pstrCN)); - snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", rsCStrGetSzStr(pstrCN)); + dbgprintf("gtls now checking auth for CN '%s'\n", cstrGetSzStr(pstrCN)); + snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", cstrGetSzStr(pstrCN)); CHKiRet(rsCStrAppendStr(pStr, lnBuf)); - CHKiRet(gtlsChkOnePeerName(pThis, rsCStrGetSzStr(pstrCN), &bFoundPositiveMatch)); + CHKiRet(gtlsChkOnePeerName(pThis, cstrGetSzStr(pstrCN), &bFoundPositiveMatch)); } } if(!bFoundPositiveMatch) { dbgprintf("invalid peer name, not permitted to talk to it\n"); if(pThis->bReportAuthErr == 1) { - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer name not authorized - " "not permitted to talk to it. Names: %s", - rsCStrGetSzStr(pStr)); + cstrGetSzStr(pStr)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -1010,13 +1012,13 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) errmsg.LogError(0, NO_ERRCODE, "not permitted to talk to peer, certificate invalid: %s", pszErrCause); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_INVALID); } /* get current time for certificate validation */ - if(time(&ttNow) == -1) + if(datetime.GetTime(&ttNow) == -1) ABORT_FINALIZE(RS_RET_SYS_ERR); /* as it looks, we need to validate the expiration dates ourselves... @@ -1032,8 +1034,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert > ttNow) { errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "not permitted to talk to peer: certificate %d not yet active", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_NOT_YET_ACTIVE); } @@ -1043,8 +1045,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert < ttNow) { errmsg.LogError(0, RS_RET_CERT_EXPIRED, "not permitted to talk to peer: certificate %d expired", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_EXPIRED); } gnutls_x509_crt_deinit(cert); @@ -1325,7 +1327,10 @@ finalize_it: static void CheckConnection(nsd_t __attribute__((unused)) *pNsd) { - /* dummy, do nothing */ + nsd_gtls_t *pThis = (nsd_gtls_t*) pNsd; + ISOBJ_TYPE_assert(pThis, nsd_gtls); + + nsd_ptcp.CheckConnection(pThis->pTcp); } @@ -1479,7 +1484,7 @@ Rcv(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) if(pThis->pszRcvBuf == NULL) { /* we have no buffer, so we need to malloc one */ - CHKmalloc(pThis->pszRcvBuf = malloc(NSD_GTLS_MAX_RCVBUF)); + CHKmalloc(pThis->pszRcvBuf = MALLOC(NSD_GTLS_MAX_RCVBUF)); pThis->lenRcvBuf = -1; } @@ -1560,6 +1565,16 @@ finalize_it: RETiRet; } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + return nsd_ptcp.EnableKeepAlive(pNsd); +} + + /* open a connection to a remote host (server). With GnuTLS, we always * open a plain tcp socket and then, if in TLS mode, do a handshake on it. @@ -1669,6 +1684,7 @@ CODESTARTobjQueryInterface(nsd_gtls) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->GetRemAddr = GetRemAddr; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_gtls) @@ -1683,6 +1699,7 @@ CODESTARTObjClassExit(nsd_gtls) objRelease(nsd_ptcp, LM_NSD_PTCP_FILENAME); objRelease(net, LM_NET_FILENAME); objRelease(glbl, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); ENDObjClassExit(nsd_gtls) @@ -1694,6 +1711,7 @@ ENDObjClassExit(nsd_gtls) BEGINObjClassInit(nsd_gtls, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); CHKiRet(objUse(nsd_ptcp, LM_NSD_PTCP_FILENAME)); diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index cc531ca0..744955c7 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -48,6 +48,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nsdsel_ptcp.h" +#include "nsdpoll_ptcp.h" #include "nsd_ptcp.h" MODULE_TYPE_LIB @@ -296,12 +297,12 @@ FillRemHost(nsd_ptcp_t *pThis, struct sockaddr *pAddr) * memory consumption) */ len = strlen((char*)szIP) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostIP = malloc(len)) == NULL) + if((pThis->pRemHostIP = MALLOC(len)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pRemHostIP, szIP, len); len = strlen((char*)szHname) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostName = malloc(len)) == NULL) { + if((pThis->pRemHostName = MALLOC(len)) == NULL) { free(pThis->pRemHostIP); /* prevent leak */ pThis->pRemHostIP = NULL; ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -562,6 +563,7 @@ finalize_it: static rsRetVal Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) { + char errStr[1024]; DEFiRet; nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; ISOBJ_TYPE_assert(pThis, nsd_ptcp); @@ -571,7 +573,9 @@ Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) if(*pLenBuf == 0) { ABORT_FINALIZE(RS_RET_CLOSED); } else if (*pLenBuf < 0) { - ABORT_FINALIZE(RS_RET_ERR); + rs_strerror_r(errno, errStr, sizeof(errStr)); + dbgprintf("error during recv on NSD %p: %s\n", pNsd, errStr); + ABORT_FINALIZE(RS_RET_RCV_ERR); } finalize_it: @@ -614,6 +618,34 @@ finalize_it: } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; + int ret; + int optval; + socklen_t optlen; + DEFiRet; + ISOBJ_TYPE_assert(pThis, nsd_ptcp); + + optval = 1; + optlen = sizeof(optval); + ret = setsockopt(pThis->sock, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen); + if(ret < 0) { + dbgprintf("EnableKeepAlive socket call returns error %d\n", ret); + ABORT_FINALIZE(RS_RET_ERR); + } + + dbgprintf("KEEPALIVE enabled for nsd %p\n", pThis); + +finalize_it: + RETiRet; +} + + /* open a connection to a remote host (server). * rgerhards, 2008-03-19 */ @@ -754,6 +786,7 @@ CODESTARTobjQueryInterface(nsd_ptcp) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->CheckConnection = CheckConnection; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_ptcp) @@ -792,6 +825,7 @@ ENDObjClassInit(nsd_ptcp) BEGINmodExit CODESTARTmodExit + nsdpoll_ptcpClassExit(); nsdsel_ptcpClassExit(); nsd_ptcpClassExit(); ENDmodExit @@ -810,6 +844,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(nsd_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ CHKiRet(nsdsel_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + CHKiRet(nsdpoll_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ ENDmodInit /* vi:set ai: */ diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c new file mode 100644 index 00000000..51006707 --- /dev/null +++ b/runtime/nsdpoll_ptcp.c @@ -0,0 +1,288 @@ +/* nsdpoll_ptcp.c + * + * An implementation of the nsd epoll() interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#ifdef HAVE_EPOLL_CREATE /* this module requires epoll! */ + +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "errmsg.h" +#include "srUtils.h" +#include "nspoll.h" +#include "nsd_ptcp.h" +#include "nsdpoll_ptcp.h" +#include "unlimited_select.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + + +/* -START------------------------- helpers for event list ------------------------------------ */ + +/* add new entry to list. We assume that the fd is not already present and DO NOT check this! + * Returns newly created entry in pEvtLst. + * Note that we currently need to use level-triggered mode, because the upper layers do not work + * in parallel. As such, in edge-triggered mode we may not get notified, because new data comes + * in after we have read everything that was present. To use ET mode, we need to change the upper + * peers so that they immediately start a new wait before processing the data read. That obviously + * requires more elaborate redesign and we postpone this until the current more simplictic mode has + * been proven OK in practice. + * rgerhards, 2009-11-18 + */ +static inline rsRetVal +addEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, int mode, nsd_ptcp_t *pSock, nsdpoll_epollevt_lst_t **pEvtLst) { + nsdpoll_epollevt_lst_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (nsdpoll_epollevt_lst_t*) malloc(sizeof(nsdpoll_epollevt_lst_t))); + pNew->id = id; + pNew->pUsr = pUsr; + pNew->pSock = pSock; + pNew->event.events = 0; /* TODO: at some time we should be able to use EPOLLET */ + if(mode & NSDPOLL_IN) + pNew->event.events |= EPOLLIN; + if(mode & NSDPOLL_OUT) + pNew->event.events |= EPOLLOUT; + pNew->event.data.u64 = (uint64) pNew; + pNew->pNext = pThis->pRoot; + pThis->pRoot = pNew; + *pEvtLst = pNew; + +finalize_it: + RETiRet; +} + + +/* find and unlink the entry identified by id/pUsr from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +unlinkEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, nsdpoll_epollevt_lst_t **ppEvtLst) { + nsdpoll_epollevt_lst_t *pEvtLst; + nsdpoll_epollevt_lst_t *pPrev = NULL; + DEFiRet; + + pEvtLst = pThis->pRoot; + while(pEvtLst != NULL && !(pEvtLst->id == id && pEvtLst->pUsr == pUsr)) { + pPrev = pEvtLst; + pEvtLst = pEvtLst->pNext; + } + if(pEvtLst == NULL) + ABORT_FINALIZE(RS_RET_NOT_FOUND); + + *ppEvtLst = pEvtLst; + + /* unlink */ + if(pPrev == NULL) + pThis->pRoot = pEvtLst->pNext; + else + pPrev->pNext = pEvtLst->pNext; + +finalize_it: + RETiRet; +} + + +/* destruct the provided element. It must already be unlinked from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) { + DEFiRet; + free(*ppEvtLst); + *ppEvtLst = NULL; + RETiRet; +} + + +/* -END--------------------------- helpers for event list ------------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */ +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); + pThis->efd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n"); + pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */ +# endif + if(pThis->efd < 0) { + DBGPRINTF("epoll_create1() could not create fd\n"); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } +finalize_it: +ENDobjConstruct(nsdpoll_ptcp) + + +/* destructor for the nsdpoll_ptcp object */ +BEGINobjDestruct(nsdpoll_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nsdpoll_ptcp) +ENDobjDestruct(nsdpoll_ptcp) + + +/* Modify socket set */ +static rsRetVal +Ctl(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; + nsdpoll_epollevt_lst_t *pEventLst; + int errSave; + char errStr[512]; + DEFiRet; + + if(op == NSDPOLL_ADD) { + dbgprintf("adding nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(addEvent(pThis, id, pUsr, mode, pSock, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_ADD, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + } + } else if(op == NSDPOLL_DEL) { + dbgprintf("removing nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(unlinkEvent(pThis, id, pUsr, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_DEL, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + ABORT_FINALIZE(RS_RET_ERR_EPOLL_CTL); + } + CHKiRet(delEvent(&pEventLst)); + } else { + dbgprintf("program error: invalid NSDPOLL_mode %d - ignoring request\n", op); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Wait for io to become ready. After the successful call, idRdy contains the + * id set by the caller for that i/o event, ppUsr is a pointer to a location + * where the user pointer shall be stored. + * TODO: this is a trivial implementation that only polls one event at a time. We + * may later extend it to poll for multiple events, what would cause less + * overhead. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Wait(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsdpoll_epollevt_lst_t *pOurEvt; + struct epoll_event event; + int nfds; + DEFiRet; + + assert(idRdy != NULL); + assert(ppUsr != NULL); + + nfds = epoll_wait(pThis->efd, &event, 1, timeout); + if(nfds == -1) { + if(errno == EINTR) { + ABORT_FINALIZE(RS_RET_EINTR); + } else { + DBGPRINTF("epoll() returned with error code %d\n", errno); + ABORT_FINALIZE(RS_RET_ERR_EPOLL); + } + } else if(nfds == 0) { + ABORT_FINALIZE(RS_RET_TIMEOUT); + } + + /* we got a valid event, so tell the caller... */ + pOurEvt = (nsdpoll_epollevt_lst_t*) event.data.u64; + *idRdy = pOurEvt->id; + *ppUsr = pOurEvt->pUsr; + +finalize_it: + RETiRet; +} + + +/* ------------------------------ end support for the epoll() interface ------------------------------ */ + + +/* queryInterface function */ +BEGINobjQueryInterface(nsdpoll_ptcp) +CODESTARTobjQueryInterface(nsdpoll_ptcp) + if(pIf->ifVersion != nsdCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpConstruct; + pIf->Destruct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpDestruct; + pIf->Ctl = Ctl; + pIf->Wait = Wait; +finalize_it: +ENDobjQueryInterface(nsdpoll_ptcp) + + +/* exit our class + */ +BEGINObjClassExit(nsdpoll_ptcp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdpoll_ptcp) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(nsdpoll_ptcp) + + +/* Initialize the nsdpoll_ptcp class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nsdpoll_ptcp, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nsdpoll_ptcp) +#endif /* #ifdef HAVE_EPOLL_CREATE this module requires epoll! */ + +/* vi:set ai: + */ diff --git a/runtime/nsdpoll_ptcp.h b/runtime/nsdpoll_ptcp.h new file mode 100644 index 00000000..cea2823d --- /dev/null +++ b/runtime/nsdpoll_ptcp.h @@ -0,0 +1,60 @@ +/* An implementation of the nsd poll interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSDPOLL_PTCP_H +#define INCLUDED_NSDPOLL_PTCP_H + +#include "nsd.h" +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif +typedef nsdpoll_if_t nsdpoll_ptcp_if_t; /* we just *implement* this interface */ +/* a helper object to keep track of the epoll event records + * Note that we need to keep track of that list because we need to + * free the events when they are no longer needed. + */ +typedef struct nsdpoll_epollevt_lst_s nsdpoll_epollevt_lst_t; +struct nsdpoll_epollevt_lst_s { +#if HAVE_SYS_EPOLL_H + epoll_event_t event; +#endif + int id; + void *pUsr; + nsd_ptcp_t *pSock; /* our associated netstream driver data */ + nsdpoll_epollevt_lst_t *pNext; +}; + +/* the nsdpoll_ptcp object */ +struct nsdpoll_ptcp_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int efd; /* file descriptor used by epoll */ + nsdpoll_epollevt_lst_t *pRoot; /* Root of the epoll event list */ +}; + +/* interface is defined in nsd.h, we just implement it! */ +#define nsdpoll_ptcpCURR_IF_VERSION nsdCURR_IF_VERSION + +/* prototypes */ +PROTOTYPEObj(nsdpoll_ptcp); + +#endif /* #ifndef INCLUDED_NSDPOLL_PTCP_H */ diff --git a/runtime/nsdsel_ptcp.c b/runtime/nsdsel_ptcp.c index 41b85e0c..e2cfca7c 100644 --- a/runtime/nsdsel_ptcp.c +++ b/runtime/nsdsel_ptcp.c @@ -36,6 +36,7 @@ #include "errmsg.h" #include "nsd_ptcp.h" #include "nsdsel_ptcp.h" +#include "unlimited_select.h" /* static data */ DEFobjStaticHelpers @@ -47,14 +48,23 @@ DEFobjCurrIf(glbl) */ BEGINobjConstruct(nsdsel_ptcp) /* be sure to specify the object type also in END macro! */ pThis->maxfds = 0; +#ifdef USE_UNLIMITED_SELECT + pThis->pReadfds = calloc(1, glbl.GetFdSetSize()); + pThis->pWritefds = calloc(1, glbl.GetFdSetSize()); +#else FD_ZERO(&pThis->readfds); FD_ZERO(&pThis->writefds); +#endif ENDobjConstruct(nsdsel_ptcp) /* destructor for the nsdsel_ptcp object */ BEGINobjDestruct(nsdsel_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(nsdsel_ptcp) +#ifdef USE_UNLIMITED_SELECT + freeFdSet(pThis->pReadfds); + freeFdSet(pThis->pWritefds); +#endif ENDobjDestruct(nsdsel_ptcp) @@ -65,20 +75,27 @@ Add(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pSock, nsd_ptcp); ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); switch(waitOp) { case NSDSEL_RD: - FD_SET(pSock->sock, &pThis->readfds); + FD_SET(pSock->sock, pReadfds); break; case NSDSEL_WR: - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - FD_SET(pSock->sock, &pThis->readfds); - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pReadfds); + FD_SET(pSock->sock, pWritefds); break; } @@ -98,6 +115,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) DEFiRet; int i; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); assert(piNumReady != NULL); @@ -106,13 +130,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) // TODO: name in dbgprintf! dbgprintf("--------<NSDSEL_PTCP> calling select, active fds (max %d): ", pThis->maxfds); for(i = 0; i <= pThis->maxfds; ++i) - if(FD_ISSET(i, &pThis->readfds) || FD_ISSET(i, &pThis->writefds)) + if(FD_ISSET(i, pReadfds) || FD_ISSET(i, pWritefds)) dbgprintf("%d ", i); dbgprintf("\n"); } /* now do the select */ - *piNumReady = select(pThis->maxfds+1, &pThis->readfds, &pThis->writefds, NULL, NULL); + *piNumReady = select(pThis->maxfds+1, pReadfds, pWritefds, NULL, NULL); RETiRet; } @@ -125,6 +149,13 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); ISOBJ_TYPE_assert(pSock, nsd_ptcp); @@ -132,14 +163,14 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) switch(waitOp) { case NSDSEL_RD: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds); break; case NSDSEL_WR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds) - | FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds) + | FD_ISSET(pSock->sock, pWritefds); break; } diff --git a/runtime/nsdsel_ptcp.h b/runtime/nsdsel_ptcp.h index 6c0c7fa7..f9ec8210 100644 --- a/runtime/nsdsel_ptcp.h +++ b/runtime/nsdsel_ptcp.h @@ -31,8 +31,13 @@ typedef nsdsel_if_t nsdsel_ptcp_if_t; /* we just *implement* this interface */ struct nsdsel_ptcp_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ int maxfds; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds; + fd_set *pWritefds; +#else fd_set readfds; fd_set writefds; +#endif }; /* interface is defined in nsd.h, we just implement it! */ diff --git a/runtime/nspoll.c b/runtime/nspoll.c new file mode 100644 index 00000000..f287cd4e --- /dev/null +++ b/runtime/nspoll.c @@ -0,0 +1,198 @@ +/* nspoll.c + * + * This is an io waiter interface utilizing the much-more-efficient poll/epoll API. + * Note that it may not always be available for a given driver. If so, that is reported + * back to the upper peer which then should consult a nssel-based io waiter. + * + * Work on this module begun 2009-11-18 by Rainer Gerhards. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include "rsyslog.h" +#include "obj.h" +#include "module-template.h" +#include "netstrm.h" +#include "nspoll.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) + + +/* load our low-level driver. This must be done before any + * driver-specific functions (allmost all...) can be carried + * out. Note that the driver's .ifIsLoaded is correctly + * initialized by calloc() and we depend on that. Please note that + * we do some name-mangeling. We know that each nsd driver also needs + * a nspoll driver. So we simply append "sel" to the nsd driver name: This, + * of course, means that the driver name must match these rules, but that + * shouldn't be a real problem. + * WARNING: this code is mostly identical to similar code in + * netstrms.c - TODO: abstract it and move it to some common place. + * rgerhards, 2008-04-28 + */ +static rsRetVal +loadDrvr(nspoll_t *pThis) +{ + DEFiRet; + uchar *pBaseDrvrName; + uchar szDrvrName[48]; /* 48 shall be large enough */ + + pBaseDrvrName = pThis->pBaseDrvrName; + if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdpoll_%s", pBaseDrvrName) == sizeof(szDrvrName)) + ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); + CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); + + pThis->Drvr.ifVersion = nsdCURR_IF_VERSION; + /* The pDrvrName+2 below is a hack to obtain the object name. It + * safes us to have yet another variable with the name without "lm" in + * front of it. If we change the module load interface, we may re-think + * about this hack, but for the time being it is efficient and clean + * enough. -- rgerhards, 2008-04-18 + */ +RUNLOG_VAR("%s", szDrvrName+2); + CHKiRet(obj.UseObj(__FILE__, szDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr)); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pDrvrName != NULL) + free(pThis->pDrvrName); + pThis->pDrvrName = NULL; + } + RETiRet; +} + + +/* Standard-Constructor */ +BEGINobjConstruct(nspoll) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(nspoll) + + +/* destructor for the nspoll object */ +BEGINobjDestruct(nspoll) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nspoll) + if(pThis->pDrvrData != NULL) + pThis->Drvr.Destruct(&pThis->pDrvrData); + + /* and now we must release our driver, if we got one. We use the presence of + * a driver name string as load indicator (because we also need that string + * to release the driver + */ + if(pThis->pDrvrName != NULL) { + obj.ReleaseObj(__FILE__, pThis->pDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr); + free(pThis->pDrvrName); + } +ENDobjDestruct(nspoll) + + +/* ConstructionFinalizer */ +static rsRetVal +ConstructFinalize(nspoll_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); +RUNLOG_STR("trying to load epoll driver\n"); + CHKiRet(loadDrvr(pThis)); + CHKiRet(pThis->Drvr.Construct(&pThis->pDrvrData)); +finalize_it: +dbgprintf("XXX: done trying to load epoll driver, state %d\n", iRet); + RETiRet; +} + + +/* Carries out the actual wait (all done in lower layers) + */ +static rsRetVal +Wait(nspoll_t *pThis, int timeout, int *idRdy, void **ppUsr) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + assert(idRdy != NULL); + iRet = pThis->Drvr.Wait(pThis->pDrvrData, timeout, idRdy, ppUsr); + RETiRet; +} + + +/* semantics like the epoll_ctl() function, does the same thing. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Ctl(nspoll_t *pThis, netstrm_t *pStrm, int id, void *pUsr, int mode, int op) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + iRet = pThis->Drvr.Ctl(pThis->pDrvrData, pStrm->pDrvrData, id, pUsr, mode, op); + RETiRet; +} + + +/* queryInterface function */ +BEGINobjQueryInterface(nspoll) +CODESTARTobjQueryInterface(nspoll) + if(pIf->ifVersion != nspollCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = nspollConstruct; + pIf->ConstructFinalize = ConstructFinalize; + pIf->Destruct = nspollDestruct; + pIf->Wait = Wait; + pIf->Ctl = Ctl; +finalize_it: +ENDobjQueryInterface(nspoll) + + +/* exit our class + */ +BEGINObjClassExit(nspoll, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nspoll) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(nspoll) + + +/* Initialize the nspoll class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nspoll, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + DBGPRINTF("doing nspollClassInit\n"); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nspoll) +/* vi:set ai: + */ diff --git a/runtime/nspoll.h b/runtime/nspoll.h new file mode 100644 index 00000000..a77759c0 --- /dev/null +++ b/runtime/nspoll.h @@ -0,0 +1,65 @@ +/* Definitions for the nspoll io activity waiter + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSPOLL_H +#define INCLUDED_NSPOLL_H + +#include "netstrms.h" + +/* some operations to be portable when we do not have epoll() available */ +#define NSDPOLL_ADD 1 +#define NSDPOLL_DEL 2 + +/* and some mode specifiers for waiting on input/output */ +#define NSDPOLL_IN 1 /* EPOLLIN */ +#define NSDPOLL_OUT 2 /* EPOLLOUT */ +/* next is 4, 8, 16, ... - must be bit values, as they are ored! */ + +/* the nspoll object */ +struct nspoll_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + nsd_t *pDrvrData; /**< the driver's data elements */ + uchar *pBaseDrvrName; /**< nsd base driver name to use, or NULL if system default */ + uchar *pDrvrName; /**< full base driver name (set when driver is loaded) */ + nsdpoll_if_t Drvr; /**< our stream driver */ +}; + + +/* interface */ +BEGINinterface(nspoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nspoll_t **ppThis); + rsRetVal (*ConstructFinalize)(nspoll_t *pThis); + rsRetVal (*Destruct)(nspoll_t **ppThis); + rsRetVal (*Wait)(nspoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); + rsRetVal (*Ctl)(nspoll_t *pNsdpoll, netstrm_t *pStrm, int id, void *pUsr, int mode, int op); + rsRetVal (*IsEPollSupported)(void); /* static method */ +ENDinterface(nspoll) +#define nspollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + +/* prototypes */ +PROTOTYPEObj(nspoll); + +/* the name of our library binary */ +#define LM_NSPOLL_FILENAME LM_NETSTRMS_FILENAME + +#endif /* #ifndef INCLUDED_NSPOLL_H */ diff --git a/runtime/nssel.c b/runtime/nssel.c index d11d5fe1..7c5be3a9 100644 --- a/runtime/nssel.c +++ b/runtime/nssel.c @@ -219,6 +219,7 @@ ENDObjClassExit(nssel) */ BEGINObjClassInit(nssel, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + DBGPRINTF("doing nsselClassInit\n"); CHKiRet(objUse(glbl, CORE_COMPONENT)); /* set our own handlers */ diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 914c2f2c..e1b54d4f 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -105,12 +105,13 @@ struct obj_s { /* the dummy struct that each derived class can be casted to */ # define ISOBJ_TYPE_assert(pObj, objType) \ do { \ ASSERT(pObj != NULL); \ - ASSERT((unsigned) ((obj_t*) (pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ if(strcmp((char*)(((obj_t*)pObj)->pObjInfo->pszID), #objType)) { \ dbgprintf("%s:%d ISOBJ assert failure: invalid object type, expected '%s' " \ - "actual '%s'\n", __FILE__, __LINE__, #objType, (((obj_t*)pObj)->pObjInfo->pszID)); \ + "actual '%s', cookie: %X\n", __FILE__, __LINE__, #objType, \ + (((obj_t*)pObj)->pObjInfo->pszID), ((obj_t*)(pObj))->iObjCooCKiE); \ assert(0); /* trigger assertion, messge we already have */ \ } \ + ASSERT((unsigned) ((obj_t*)(pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ } while(0) #else /* non-debug mode, no checks but much faster */ # define BEGINobjInstance obj_t objData @@ -280,7 +281,7 @@ rsRetVal objName##ClassExit(void) \ * rgerhards, 2008-01-30 */ #define BEGINobjDestruct(OBJ) \ - rsRetVal OBJ##Destruct(OBJ##_t **ppThis) \ + rsRetVal OBJ##Destruct(OBJ##_t __attribute__((unused)) **ppThis) \ { \ DEFiRet; \ int iCancelStateSave; \ @@ -292,6 +293,15 @@ rsRetVal objName##ClassExit(void) \ ISOBJ_TYPE_assert(pThis, OBJ); \ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); +/* note: there was a long-time bug in the macro below that lead to *ppThis = NULL + * only when the object was actually destructed. I discovered this issue during + * introduction of the pRcvFrom property in msg_t, but it potentially had other + * effects, too. I am not sure if some experienced instability resulted from this + * bug OR if its fix will cause harm to so-far "correctly" running code. The later + * may very well be. Thus I will change it only for the current branch and also + * the beta, but not in all old builds. Let's see how things evolve. + * rgerhards, 2009-06-30 + */ #define ENDobjDestruct(OBJ) \ goto finalize_it; /* prevent compiler warning ;) */ \ /* no more code here! */ \ @@ -299,8 +309,8 @@ rsRetVal objName##ClassExit(void) \ if(pThis != NULL) { \ obj.DestructObjSelf((obj_t*) pThis); \ free(pThis); \ - *ppThis = NULL; \ } \ + *ppThis = NULL; \ pthread_setcancelstate(iCancelStateSave, NULL); \ RETiRet; \ } @@ -314,7 +324,7 @@ rsRetVal objName##ClassExit(void) \ #define PROTOTYPEObjDebugPrint(obj) rsRetVal obj##DebugPrint(obj##_t *pThis) #define INTERFACEObjDebugPrint(obj) rsRetVal (*DebugPrint)(obj##_t *pThis) #define BEGINobjDebugPrint(obj) \ - rsRetVal obj##DebugPrint(obj##_t *pThis) \ + rsRetVal obj##DebugPrint(obj##_t __attribute__((unused)) *pThis) \ { \ DEFiRet; \ diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..45dac776 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -48,7 +48,7 @@ * * File begun on 2008-01-04 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -75,6 +75,7 @@ #include <string.h> #include <ctype.h> #include <assert.h> +#include <pthread.h> /* how many objects are supported by rsyslogd? */ #define OBJ_NUM_IDS 100 /* TODO change to a linked list? info: 16 were currently in use 2008-02-29 */ @@ -87,13 +88,18 @@ #include "modules.h" #include "errmsg.h" #include "cfsysline.h" +#include "unicode-helper.h" +#include "apc.h" +#include "datetime.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ DEFobjCurrIf(var) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +DEFobjCurrIf(strm) static objInfo_t *arrObjInfo[OBJ_NUM_IDS]; /* array with object information pointers */ +static pthread_mutex_t mutObjGlobalOp; /* mutex to guard global operations of the object system */ /* cookies for serialized lines */ @@ -144,8 +150,8 @@ InfoConstruct(objInfo_t **ppThis, uchar *pszID, int iObjVers, ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pThis->pszID = pszID; - pThis->lenID = strlen((char*)pszID); - pThis->pszName = (uchar*)strdup((char*)pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ + pThis->lenID = ustrlen(pszID); + pThis->pszName = ustrdup(pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ pThis->iObjVers = iObjVers; pThis->QueryIF = pQueryIF; pThis->pModInfo = pModInfo; @@ -176,8 +182,7 @@ InfoDestruct(objInfo_t **ppThis) pThis = *ppThis; assert(pThis != NULL); - if(pThis->pszName != NULL) - free(pThis->pszName); + free(pThis->pszName); free(pThis); *ppThis = NULL; @@ -205,9 +210,7 @@ DestructObjSelf(obj_t *pThis) DEFiRet; ISOBJ_assert(pThis); - if(pThis->pszName != NULL) { - free(pThis->pszName); - } + free(pThis->pszName); RETiRet; } @@ -228,20 +231,20 @@ static rsRetVal objSerializeHeader(strm_t *pStrm, obj_t *pObj, uchar *pszRecType assert(!strcmp((char*) pszRecType, "Obj") || !strcmp((char*) pszRecType, "OPB")); /* object cookie and serializer version (so far always 1) */ - CHKiRet(strmWriteChar(pStrm, COOKIE_OBJLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '1')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_OBJLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '1')); /* object type, version and string length */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWrite(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteLong(pStrm, objGetVersion(pObj))); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, objGetVersion(pObj))); /* record trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -259,7 +262,7 @@ BeginSerialize(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "Obj")); finalize_it: @@ -284,7 +287,7 @@ BeginSerializePropBag(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "OPB")); finalize_it: @@ -320,31 +323,31 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr switch(propType) { case PROPTYPE_PSZ: pszBuf = (uchar*) pUsr; - lenBuf = strlen((char*) pszBuf); + lenBuf = ustrlen(pszBuf); vType = VARTYPE_STR; break; case PROPTYPE_SHORT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((short*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((int*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_LONG: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((long*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT64: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((int64*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_CSTR: @@ -377,23 +380,23 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr } /* cookie */ - CHKiRet(strmWriteChar(pStrm, COOKIE_PROPLINE)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_PROPLINE)); /* name */ - CHKiRet(strmWrite(pStrm, pszPropName, strlen((char*)pszPropName))); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pszPropName, ustrlen(pszPropName))); + CHKiRet(strm.WriteChar(pStrm, ':')); /* type */ - CHKiRet(strmWriteLong(pStrm, (int) vType)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, (int) vType)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* length */ - CHKiRet(strmWriteLong(pStrm, lenBuf)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, lenBuf)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* data */ - CHKiRet(strmWrite(pStrm, (uchar*) pszBuf, lenBuf)); + CHKiRet(strm.Write(pStrm, (uchar*) pszBuf, lenBuf)); /* trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -410,12 +413,12 @@ EndSerialize(strm_t *pStrm) assert(pStrm != NULL); - CHKiRet(strmWriteChar(pStrm, COOKIE_ENDLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); - CHKiRet(strmWriteChar(pStrm, COOKIE_BLANKLINE)); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_ENDLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_BLANKLINE)); + CHKiRet(strm.WriteChar(pStrm, '\n')); - CHKiRet(strmRecordEnd(pStrm)); + CHKiRet(strm.RecordEnd(pStrm)); finalize_it: RETiRet; @@ -423,7 +426,7 @@ finalize_it: /* define a helper to make code below a bit cleaner (and quicker to write) */ -#define NEXTC CHKiRet(strmReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ +#define NEXTC CHKiRet(strm.ReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ /* de-serialize an embedded, non-octect-counted string. This is useful @@ -440,20 +443,20 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) assert(ppStr != NULL); - CHKiRet(rsCStrConstruct(&pStr)); + CHKiRet(cstrConstruct(&pStr)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pStr, c)); + CHKiRet(cstrAppendChar(pStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; finalize_it: if(iRet != RS_RET_OK && pStr != NULL) - rsCStrDestruct(&pStr); + cstrDestruct(&pStr); RETiRet; } @@ -508,14 +511,14 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) assert(ppCStr != NULL); assert(iLen >= 0); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); NEXTC; for(i = 0 ; i < iLen ; ++i) { - CHKiRet(rsCStrAppendChar(pCStr, c)); + CHKiRet(cstrAppendChar(pCStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* check terminator */ if(c != ':') ABORT_FINALIZE(RS_RET_INVALID_DELIMITER); @@ -524,7 +527,7 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) finalize_it: if(iRet != RS_RET_OK && pCStr != NULL) - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); RETiRet; } @@ -617,19 +620,19 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) NEXTC; if(c != COOKIE_PROPLINE) { /* oops, we've read one char that does not belong to use - unget it first */ - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); ABORT_FINALIZE(RS_RET_NO_PROPLINE); } /* get the property name first */ - CHKiRet(rsCStrConstruct(&pProp->pcsName)); + CHKiRet(cstrConstruct(&pProp->pcsName)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pProp->pcsName, c)); + CHKiRet(cstrAppendChar(pProp->pcsName, c)); NEXTC; } - CHKiRet(rsCStrFinish(pProp->pcsName)); + CHKiRet(cstrFinalize(pProp->pcsName)); /* property type */ CHKiRet(objDeserializeNumber(&i, pStrm)); @@ -718,7 +721,7 @@ static rsRetVal objDeserializeTryRecover(strm_t *pStrm) } } - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); finalize_it: dbgprintf("deserializer has possibly been able to re-sync and recover, state %d\n", iRet); @@ -803,7 +806,7 @@ Deserialize(void *ppObj, uchar *pszTypeExpected, strm_t *pStrm, rsRetVal (*fFixu } } while(iRetLocal != RS_RET_OK); - if(rsCStrSzStrCmp(pstrID, pszTypeExpected, strlen((char*)pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ + if(rsCStrSzStrCmp(pstrID, pszTypeExpected, ustrlen(pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ ABORT_FINALIZE(RS_RET_INVALID_OID); CHKiRet(FindObjInfo(pstrID, &pObjInfo)); @@ -948,13 +951,8 @@ SetName(obj_t *pThis, uchar *pszName) { DEFiRet; - if(pThis->pszName != NULL) - free(pThis->pszName); - - pThis->pszName = (uchar*) strdup((char*) pszName); - - if(pThis->pszName == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); finalize_it: RETiRet; @@ -1057,7 +1055,7 @@ RegisterObj(uchar *pszObjName, objInfo_t *pInfo) i = 0; while(!bFound && i < OBJ_NUM_IDS && arrObjInfo[i] != NULL) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } @@ -1096,7 +1094,7 @@ UnregisterObj(uchar *pszObjName) i = 0; while(!bFound && i < OBJ_NUM_IDS) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } @@ -1132,6 +1130,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* DEV debug only: dbgprintf("source file %s requests object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ + pthread_mutex_lock(&mutObjGlobalOp); if(pIf->ifIsLoaded == 1) { ABORT_FINALIZE(RS_RET_OK); /* we are already set */ @@ -1172,6 +1171,8 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 1; /* we are happy */ finalize_it: + pthread_mutex_unlock(&mutObjGlobalOp); + if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1193,15 +1194,16 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* dev debug only dbgprintf("source file %s releasing object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ + pthread_mutex_lock(&mutObjGlobalOp); if(pObjFile == NULL) FINALIZE; /* if it is not a lodable module, we do not need to do anything... */ if(pIf->ifIsLoaded == 0) { - ABORT_FINALIZE(RS_RET_OK); /* we are not loaded - this is perfectly OK... */ + FINALIZE; /* we are not loaded - this is perfectly OK... */ } else if(pIf->ifIsLoaded == 2) { pIf->ifIsLoaded = 0; /* clean up */ - ABORT_FINALIZE(RS_RET_OK); /* we had a load error and can not continue */ + FINALIZE; /* we had a load error and can not/must not continue */ } CHKiRet(rsCStrConstructFromszStr(&pStr, pObjName)); @@ -1213,6 +1215,8 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 0; /* indicated "no longer valid" */ finalize_it: + pthread_mutex_unlock(&mutObjGlobalOp); + if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1278,14 +1282,15 @@ objClassExit(void) { DEFiRet; /* release objects we no longer need */ + objRelease(strm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); /* TODO: implement the class exits! */ #if 0 - cfsyslineInit(pModInfo); - varClassInit(pModInfo); + cfsyslineExit(pModInfo); + varClassExit(pModInfo); #endif errmsgClassExit(); moduleClassExit(); @@ -1304,8 +1309,9 @@ objClassExit(void) rsRetVal objClassInit(modInfo_t *pModInfo) { - DEFiRet; + pthread_mutexattr_t mutAttr; int i; + DEFiRet; /* first, initialize the object system itself. This must be done * before any other object is created. @@ -1314,17 +1320,28 @@ objClassInit(modInfo_t *pModInfo) arrObjInfo[i] = NULL; } + /* the mutex must be recursive, because objects may call into other + * object identifieres recursively. + */ + pthread_mutexattr_init(&mutAttr); + pthread_mutexattr_settype(&mutAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&mutObjGlobalOp, &mutAttr); + /* request objects we use */ CHKiRet(objGetObjInterface(&obj)); /* get ourselves ;) */ /* init classes we use (limit to as few as possible!) */ CHKiRet(errmsgClassInit(pModInfo)); + CHKiRet(datetimeClassInit(pModInfo)); + CHKiRet(apcClassInit(pModInfo)); CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); CHKiRet(moduleClassInit(pModInfo)); + CHKiRet(strmClassInit(pModInfo)); CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); finalize_it: RETiRet; diff --git a/runtime/obj.h b/runtime/obj.h index dc04203b..419d29cc 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -68,7 +68,7 @@ #define objSerializePTR(strm, propName, propType) \ CHKiRet(obj.SerializeProp(strm, (uchar*) #propName, PROPTYPE_##propType, (void*) pThis->propName)); #define DEFobjStaticHelpers \ - static objInfo_t *pObjInfoOBJ = NULL; \ + static objInfo_t __attribute__((unused)) *pObjInfoOBJ = NULL; \ DEFobjCurrIf(obj) @@ -77,11 +77,13 @@ /* the next macro MUST be called in Constructors: */ #ifndef NDEBUG /* this means if debug... */ # define objConstructSetObjInfo(pThis) \ - ASSERT(((obj_t*) (pThis))->pObjInfo == NULL); \ ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ; \ + ((obj_t*) (pThis))->pszName = NULL; \ ((obj_t*) (pThis))->iObjCooCKiE = 0xBADEFEE #else -# define objConstructSetObjInfo(pThis) ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ +# define objConstructSetObjInfo(pThis) \ + ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ; \ + ((obj_t*) (pThis))->pszName = NULL #endif #define objDestruct(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_DESTRUCT])(&pThis) #define objSerialize(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_SERIALIZE]) diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 8dddc4b8..1d442c61 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -67,31 +67,25 @@ rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries) assert(ppThis != NULL); assert(iNumEntries >= 0); - if((pThis = calloc(1, sizeof(omodStringRequest_t))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis = calloc(1, sizeof(omodStringRequest_t))); /* got the structure, so fill it */ pThis->iNumEntries = iNumEntries; /* allocate string for template name array. The individual strings will be * allocated as the code progresses (we do not yet know the string sizes) */ - if((pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))); + /* allocate the template options array. */ - if((pThis->piTplOpts = calloc(iNumEntries, sizeof(int))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->piTplOpts = calloc(iNumEntries, sizeof(int))); -abort_it: +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis != NULL) { + OMSRdestruct(pThis); + pThis = NULL; + } + } *ppThis = pThis; RETiRet; } @@ -155,7 +149,7 @@ OMSRgetSupportedTplOpts(unsigned long *pOpts) { DEFiRet; assert(pOpts != NULL); - *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY; + *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY | OMSR_TPL_AS_MSG; RETiRet; } diff --git a/runtime/objomsr.h b/runtime/objomsr.h index 75ad0fb8..e59b774f 100644 --- a/runtime/objomsr.h +++ b/runtime/objomsr.h @@ -27,8 +27,12 @@ /* define flags for required template options */ #define OMSR_NO_RQD_TPL_OPTS 0 #define OMSR_RQD_TPL_OPT_SQL 1 +/* only one of OMSR_TPL_AS_ARRAY or _AS_MSG must be specified, if both are given + * results are unpredictable. + */ #define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */ -/* next option is 4, 8, 16, ... */ +#define OMSR_TPL_AS_MSG 4 /* introduced in 5.3.4, 2009-11-02 */ +/* next option is 8, 16, 32, ... */ struct omodStringRequest_s { /* strings requested by output module for doAction() */ int iNumEntries; /* number of array entries for data elements below */ diff --git a/runtime/parser.c b/runtime/parser.c index 212d40f3..ca31b35d 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -37,7 +37,13 @@ #include "dirty.h" #include "msg.h" #include "obj.h" +#include "datetime.h" #include "errmsg.h" +#include "parser.h" +#include "ruleset.h" +#include "unicode-helper.h" +#include "dirty.h" +#include "cfsysline.h" /* some defines */ #define DEFUPRI (LOG_USER|LOG_NOTICE) @@ -46,26 +52,167 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) +DEFobjCurrIf(ruleset) /* static data */ +/* config data */ +static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ +static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */ +static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ +static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ + +/* This is the list of all parsers known to us. + * This is also used to unload all modules on shutdown. + */ +parserList_t *pParsLstRoot = NULL; + +/* this is the list of the default parsers, to be used if no others + * are specified. + */ +parserList_t *pDfltParsLst = NULL; + + +/* intialize (but NOT allocate) a parser list. Primarily meant as a hook + * which can be used to extend the list in the future. So far, just sets + * it to NULL. + */ +static rsRetVal +InitParserList(parserList_t **pListRoot) +{ + *pListRoot = NULL; + return RS_RET_OK; +} + -/* this is a dummy class init +/* destruct a parser list. The list elements are destroyed, but the parser objects + * themselves are not modified. (That is done at a late stage during rsyslogd + * shutdown and need not be considered here.) */ -rsRetVal parserClassInit(void) +static rsRetVal +DestructParserList(parserList_t **ppListRoot) { + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = *ppListRoot; + while(pParsLst != NULL) { + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } + *ppListRoot = NULL; + return RS_RET_OK; +} + + +/* Add a parser to the list. We use a VERY simple and ineffcient algorithm, + * but it is employed only for a few milliseconds during config processing. So + * I prefer to keep it very simple and with simple data structures. Unfortunately, + * we need to preserve the order, but I don't like to add a tail pointer as that + * would require a container object. So I do the extra work to skip to the tail + * when adding elements... + * rgerhards, 2009-11-03 + */ +static rsRetVal +AddParserToList(parserList_t **ppListRoot, parser_t *pParser) +{ + parserList_t *pThis; + parserList_t *pTail; DEFiRet; - /* request objects we use */ - CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */ - CHKiRet(objUse(glbl, CORE_COMPONENT)); - CHKiRet(objUse(errmsg, CORE_COMPONENT)); -// TODO: free components! see action.c + CHKmalloc(pThis = MALLOC(sizeof(parserList_t))); + pThis->pParser = pParser; + pThis->pNext = NULL; + + if(*ppListRoot == NULL) { + pThis->pNext = *ppListRoot; + *ppListRoot = pThis; + } else { + /* find tail first */ + for(pTail = *ppListRoot ; pTail->pNext != NULL ; pTail = pTail->pNext) + /* just search, do nothing else */; + /* add at tail */ + pTail->pNext = pThis; + } + finalize_it: RETiRet; } +/* find a parser based on the provided name */ +static rsRetVal +FindParser(parser_t **ppParser, uchar *pName) +{ + parserList_t *pThis; + DEFiRet; + + for(pThis = pParsLstRoot ; pThis != NULL ; pThis = pThis->pNext) { + if(ustrcmp(pThis->pParser->pName, pName) == 0) { + *ppParser = pThis->pParser; + FINALIZE; /* found it, iRet still eq. OK! */ + } + } + + iRet = RS_RET_PARSER_NOT_FOUND; + +finalize_it: + RETiRet; +} + + +/* --- END helper functions for parser list handling --- */ + +/* Add a an already existing parser to the default list. As usual, order + * of calls is important (most importantly, that means the legacy parser, + * which can process everything, MUST be added last!). + * rgerhards, 2009-11-04 + */ +static rsRetVal +AddDfltParser(uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + CHKiRet(FindParser(&pParser, pName)); + CHKiRet(AddParserToList(&pDfltParsLst, pParser)); + dbgprintf("Parser '%s' added to default parser set.\n", pName); + +finalize_it: + RETiRet; +} + + + +BEGINobjConstruct(parser) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(parser) + +/* ConstructionFinalizer. The most important chore is to add the parser object + * to our global list of available parsers. + * rgerhards, 2009-11-03 + */ +rsRetVal parserConstructFinalize(parser_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + CHKiRet(AddParserToList(&pParsLstRoot, pThis)); + DBGPRINTF("Parser '%s' added to list of available parsers.\n", pThis->pName); + +finalize_it: + RETiRet; +} + +BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(parser) + dbgprintf("destructing parser '%s'\n", pThis->pName); + free(pThis->pName); +ENDobjDestruct(parser) + + /* uncompress a received message if it is compressed. * pMsg->pszRawMsg buffer is updated. * rgerhards, 2008-10-09 @@ -96,7 +243,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) */ int ret; iLenDefBuf = glbl.GetMaxLine(); - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1))); + CHKmalloc(deflateBuf = MALLOC(sizeof(uchar) * (iLenDefBuf + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, (int) (lenMsg-1)); @@ -114,10 +261,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) "Message ignored.", ret); FINALIZE; /* unconditional exit, nothing left to do... */ } - free(pMsg->pszRawMsg); - pMsg->pszRawMsg = deflateBuf; - pMsg->iLenRawMsg = iLenDefBuf; - deflateBuf = NULL; /* logically "freed" - caller is now responsible */ + MsgSetRawMsg(pMsg, (char*)deflateBuf, iLenDefBuf); } finalize_it: if(deflateBuf != NULL) @@ -156,7 +300,7 @@ finalize_it: * rgerhards, 2007-09-14 */ static inline rsRetVal -sanitizeMessage(msg_t *pMsg) +SanitizeMsg(msg_t *pMsg) { DEFiRet; uchar *pszMsg; @@ -165,12 +309,12 @@ sanitizeMessage(msg_t *pMsg) size_t iSrc; size_t iDst; size_t iMaxLine; + size_t maxDest; + sbool bUpdatedLen = FALSE; + uchar szSanBuf[32*1024]; /* buffer used for sanitizing a string */ assert(pMsg != NULL); - -# ifdef USE_NETZIP - CHKiRet(uncompressMessage(pMsg)); -# endif + assert(pMsg->iLenRawMsg > 0); pszMsg = pMsg->pszRawMsg; lenMsg = pMsg->iLenRawMsg; @@ -178,6 +322,7 @@ sanitizeMessage(msg_t *pMsg) /* remove NUL character at end of message (see comment in function header) */ if(pszMsg[lenMsg-1] == '\0') { DBGPRINTF("dropped NUL at very end of message\n"); + bUpdatedLen = TRUE; lenMsg--; } @@ -188,89 +333,112 @@ sanitizeMessage(msg_t *pMsg) */ if(bDropTrailingLF && pszMsg[lenMsg-1] == '\n') { DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); + bUpdatedLen = TRUE; lenMsg--; } + /* it is much quicker to sweep over the message and see if it actually + * needs sanitation than to do the sanitation in any case. So we first do + * this and terminate when it is not needed - which is expectedly the case + * for the vast majority of messages. -- rgerhards, 2009-06-15 + * Note that we do NOT check here if tab characters are to be escaped or + * not. I expect this functionality to be seldomly used and thus I do not + * like to pay the performance penalty. So the penalty is only with those + * that actually use it, because we may call the sanitizer without actual + * need below (but it then still will work perfectly well!). -- rgerhards, 2009-11-27 + */ + int bNeedSanitize = 0; + for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { + if(iscntrl(pszMsg[iSrc])) { + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + bNeedSanitize = 1; + break; + } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + bNeedSanitize = 1; + break; + } + } + + if(!bNeedSanitize) { + if(bUpdatedLen == TRUE) + MsgSetRawMsgSize(pMsg, lenMsg); + FINALIZE; + } + /* now copy over the message and sanitize it */ - /* TODO: can we get cheaper memory alloc? {alloca()?}*/ iMaxLine = glbl.GetMaxLine(); - CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + maxDest = lenMsg * 4; /* message can grow at most four-fold */ + if(maxDest > iMaxLine) + maxDest = iMaxLine; /* but not more than the max size! */ + if(maxDest < sizeof(szSanBuf)) + pDst = szSanBuf; + else + CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; - while(iSrc < lenMsg && iDst < iMaxLine) { - if(pszMsg[iSrc] == '\0') { /* guard against \0 characters... */ - /* changed to the sequence (somewhat) proposed in - * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 - */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ - pDst[iDst++] = cCCEscapeChar; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - } /* if we do not have space, we simply ignore the '\0'... */ - /* log an error? Very questionable... rgerhards, 2006-11-30 */ - /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ - } else if(bEscapeCCOnRcv && iscntrl((int) pszMsg[iSrc])) { - /* we are configured to escape control characters. Please note - * that this most probably break non-western character sets like - * Japanese, Korean or Chinese. rgerhards, 2007-07-17 - * Note: sysklogd logs octal values only for DEL and CCs above 127. - * For others, it logs ^n where n is the control char converted to an - * alphabet character. We like consistency and thus escape it to octal - * in all cases. If someone complains, we may change the mode. At least - * we known now what's going on. - * rgerhards, 2007-07-17 + while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ + if(iscntrl((int) pszMsg[iSrc]) && (pszMsg[iSrc] != '\t' || bEscapeTab)) { + /* note: \0 must always be escaped, the rest of the code currently + * can not handle it! -- rgerhards, 2009-08-26 */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + /* we are configured to escape control characters. Please note + * that this most probably break non-western character sets like + * Japanese, Korean or Chinese. rgerhards, 2007-07-17 + */ pDst[iDst++] = cCCEscapeChar; pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); - } /* again, if we do not have space, we ignore the char - see comment at '\0' */ + } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + /* In this case, we also do the conversion. Note that this most + * probably breaks European languages. -- rgerhards, 2010-01-27 + */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); } else { pDst[iDst++] = pszMsg[iSrc]; } ++iSrc; } - pDst[iDst] = '\0'; /* space *is* reserved for this! */ - /* we have a sanitized string. Let's save it now */ - free(pMsg->pszRawMsg); - if((pMsg->pszRawMsg = malloc((iDst+1) * sizeof(uchar))) == NULL) { - /* when we get no new buffer, we use what we already have ;) */ - pMsg->pszRawMsg = pDst; - } else { - /* trim buffer */ - memcpy(pMsg->pszRawMsg, pDst, iDst+1); - free(pDst); /* too big! */ - pMsg->iLenRawMsg = iDst; - } + MsgSetRawMsg(pMsg, (char*)pDst, iDst); /* save sanitized string */ + + if(pDst != szSanBuf) + free(pDst); finalize_it: RETiRet; } -/* Parse a received message. The object's rawmsg property is taken and - * parsed according to the relevant standards. This can later be - * extended to support configured parsers. - * rgerhards, 2008-10-09 +/* A standard parser to parse out the PRI. This is made available in + * this module as it is expected that allmost all parsers will need + * that functionality and so they do not need to implement it themsleves. */ -rsRetVal parseMsg(msg_t *pMsg) +static inline rsRetVal +ParsePRI(msg_t *pMsg) { - DEFiRet; - uchar *msg; int pri; - - CHKiRet(sanitizeMessage(pMsg)); - - /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ - DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); + uchar *msg; + int lenMsg; + int iPriText; + DEFiRet; /* pull PRI */ - pri = DEFUPRI; + lenMsg = pMsg->iLenRawMsg; msg = pMsg->pszRawMsg; + pri = DEFUPRI; + iPriText = 0; if(*msg == '<') { + /* while we process the PRI, we also fill the PRI textual representation + * inside the msg object. This may not be ideal from an OOP point of view, + * but it offers us performance... + */ pri = 0; - while(isdigit((int) *++msg)) { + while(--lenMsg > 0 && isdigit((int) *++msg)) { pri = 10 * pri + (*msg - '0'); } if(*msg == '>') @@ -280,36 +448,253 @@ rsRetVal parseMsg(msg_t *pMsg) } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - MsgSetUxTradMsg(pMsg, (char*) msg); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); + RETiRet; +} - if(pMsg->bParseHOSTNAME == 0) - MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); - /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have - * a traditional syslog message or one formatted according to syslog-protocol. - * We need to apply different parsers depending on that. We use the - * -protocol VERSION field for the detection. +/* Parse a received message. The object's rawmsg property is taken and + * parsed according to the relevant standards. This can later be + * extended to support configured parsers. + * rgerhards, 2008-10-09 + */ +static rsRetVal +ParseMsg(msg_t *pMsg) +{ + rsRetVal localRet; + parserList_t *pParserList; + parser_t *pParser; + sbool bIsSanitized; + sbool bPRIisParsed; + static int iErrMsgRateLimiter = 0; + DEFiRet; + + if(pMsg->iLenRawMsg == 0) + ABORT_FINALIZE(RS_RET_EMPTY_MSG); + +# ifdef USE_NETZIP + CHKiRet(uncompressMessage(pMsg)); +# endif + + /* we take the risk to print a non-sanitized string, because this is the best we can get + * (and that functionality is too important for debugging to drop it...). + */ + DBGPRINTF("msg parser: flags %x, from '%s', msg '%.50s'\n", pMsg->msgFlags, + (pMsg->msgFlags & NEEDS_DNSRESOL) ? UCHAR_CONSTANT("~NOTRESOLVED~") : getRcvFrom(pMsg), + pMsg->pszRawMsg); + + /* we now need to go through our list of parsers and see which one is capable of + * parsing the message. Note that the first parser that requires message sanitization + * will cause it to happen. After that, access to the unsanitized message is no + * loger possible. */ - if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); - setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + pParserList = ruleset.GetParserList(pMsg); + if(pParserList == NULL) { + pParserList = pDfltParsLst; + } + DBGPRINTF("parse using parser list %p%s.\n", pParserList, + (pParserList == pDfltParsLst) ? " (the default list)" : ""); + + bIsSanitized = FALSE; + bPRIisParsed = FALSE; + while(pParserList != NULL) { + pParser = pParserList->pParser; + if(pParser->bDoSanitazion && bIsSanitized == FALSE) { + CHKiRet(SanitizeMsg(pMsg)); + if(pParser->bDoPRIParsing && bPRIisParsed == FALSE) { + CHKiRet(ParsePRI(pMsg)); + bPRIisParsed = TRUE; + } + bIsSanitized = TRUE; } - } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); - setProtocolVersion(pMsg, 0); - if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + localRet = pParser->pModule->mod.pm.parse(pMsg); + dbgprintf("Parser '%s' returned %d\n", pParser->pName, localRet); + if(localRet != RS_RET_COULD_NOT_PARSE) + break; + pParserList = pParserList->pNext; + } + + /* We need to log a warning message and drop the message if we did not find a parser. + * Note that we log at most the first 1000 message, as this may very well be a problem + * that causes a message generation loop. We do not synchronize that counter, it doesn't + * matter if we log a handful messages more than we should... + */ + if(localRet != RS_RET_OK) { + if(++iErrMsgRateLimiter > 1000) { + errmsg.LogError(0, localRet, "Error: one message could not be processed by " + "any parser, message is being discarded (start of raw msg: '%.50s')", + pMsg->pszRawMsg); } + DBGPRINTF("No parser could process the message (state %d), we need to discard it.\n", localRet); + ABORT_FINALIZE(localRet); } - /* finalize message object */ + /* "finalize" message object */ pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */ - MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */ finalize_it: RETiRet; } + +/* set the parser name - string is copied over, call can continue to use it, + * but must free it if desired. + */ +static rsRetVal +SetName(parser_t *pThis, uchar *name) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + assert(name != NULL); + + if(pThis->pName != NULL) { + free(pThis->pName); + pThis->pName = NULL; + } + + CHKmalloc(pThis->pName = ustrdup(name)); + +finalize_it: + RETiRet; +} + + +/* set a pointer to "our" module. Note that no module + * pointer must already be set. + */ +static rsRetVal +SetModPtr(parser_t *pThis, modInfo_t *pMod) +{ + ISOBJ_TYPE_assert(pThis, parser); + assert(pMod != NULL); + assert(pThis->pModule == NULL); + pThis->pModule = pMod; + return RS_RET_OK; +} + + +/* Specify if we should do standard message sanitazion before we pass the data + * down to the parser. + */ +static rsRetVal +SetDoSanitazion(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoSanitazion = bDoIt; + return RS_RET_OK; +} + + +/* Specify if we should do standard PRI parsing before we pass the data + * down to the parser module. + */ +static rsRetVal +SetDoPRIParsing(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoPRIParsing = bDoIt; + return RS_RET_OK; +} + + +/* queryInterface function-- rgerhards, 2009-11-03 + */ +BEGINobjQueryInterface(parser) +CODESTARTobjQueryInterface(parser) + if(pIf->ifVersion != parserCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = parserConstruct; + pIf->ConstructFinalize = parserConstructFinalize; + pIf->Destruct = parserDestruct; + pIf->SetName = SetName; + pIf->SetModPtr = SetModPtr; + pIf->SetDoSanitazion = SetDoSanitazion; + pIf->SetDoPRIParsing = SetDoPRIParsing; + pIf->ParseMsg = ParseMsg; + pIf->SanitizeMsg = SanitizeMsg; + pIf->InitParserList = InitParserList; + pIf->DestructParserList = DestructParserList; + pIf->AddParserToList = AddParserToList; + pIf->AddDfltParser = AddDfltParser; + pIf->FindParser = FindParser; +finalize_it: +ENDobjQueryInterface(parser) + + + +/* Reset config variables to default values. + * rgerhards, 2007-07-17 + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + cCCEscapeChar = '#'; + bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bEscape8BitChars = 0; /* default is to escape control characters */ + bEscapeTab = 1; /* default is to escape control characters */ + bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ + + return RS_RET_OK; +} + +/* This destroys the master parserlist and all of its parser entries. MUST only be + * done when the module is shut down. Parser modules are NOT unloaded, rsyslog + * does that at a later stage for all dynamically loaded modules. + */ +static void +destroyMasterParserList(void) +{ + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = pParsLstRoot; + while(pParsLst != NULL) { + parserDestruct(&pParsLst->pParser); + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } +} + +/* Exit our class. + * rgerhards, 2009-11-04 + */ +BEGINObjClassExit(parser, OBJ_IS_CORE_MODULE) /* class, version */ + DestructParserList(&pDfltParsLst); + destroyMasterParserList(); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDObjClassExit(parser) + + +/* Initialize the parser class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2009-11-02 + */ +BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + + CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); + + InitParserList(&pParsLstRoot); + InitParserList(&pDfltParsLst); +ENDObjClassInit(parser) + diff --git a/runtime/parser.h b/runtime/parser.h index cec9c083..bdd572cb 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -1,8 +1,6 @@ /* header for parser.c - * This is not yet an object, but contains all those code necessary to - * parse syslog messages. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -21,10 +19,53 @@ * * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. */ -#ifndef INCLUDED_PARSE_H -#define INCLUDED_PARSE_H +#ifndef INCLUDED_PARSER_H +#define INCLUDED_PARSER_H -extern rsRetVal parserClassInit(void); -extern rsRetVal parseMsg(msg_t*); -#endif /* #ifndef INCLUDED_PARSE_H */ +/* we create a small helper object, a list of parsers, that we can use to + * build a chain of them whereever this is needed (initially thought to be + * used in ruleset.c as well as ourselvs). + */ +struct parserList_s { + parser_t *pParser; + parserList_t *pNext; +}; + + +/* the parser object, a dummy because we have only static methods */ +struct parser_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *pName; /* name of this parser */ + modInfo_t *pModule; /* pointer to parser's module */ + sbool bDoSanitazion; /* do standard message sanitazion before calling parser? */ + sbool bDoPRIParsing; /* do standard PRI parsing before calling parser? */ +}; + +/* interfaces */ +BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(var); + rsRetVal (*Construct)(parser_t **ppThis); + rsRetVal (*ConstructFinalize)(parser_t *pThis); + rsRetVal (*Destruct)(parser_t **ppThis); + rsRetVal (*SetName)(parser_t *pThis, uchar *name); + rsRetVal (*SetModPtr)(parser_t *pThis, modInfo_t *pMod); + rsRetVal (*SetDoSanitazion)(parser_t *pThis, int); + rsRetVal (*SetDoPRIParsing)(parser_t *pThis, int); + rsRetVal (*FindParser)(parser_t **ppThis, uchar*name); + rsRetVal (*InitParserList)(parserList_t **pListRoot); + rsRetVal (*DestructParserList)(parserList_t **pListRoot); + rsRetVal (*AddParserToList)(parserList_t **pListRoot, parser_t *pParser); + /* static functions */ + rsRetVal (*ParseMsg)(msg_t *pMsg); + rsRetVal (*SanitizeMsg)(msg_t *pMsg); + rsRetVal (*AddDfltParser)(uchar *); +ENDinterface(parser) +#define parserCURR_IF_VERSION 1 /* increment whenever you change the interface above! */ + + +/* prototypes */ +PROTOTYPEObj(parser); + + +#endif /* #ifndef INCLUDED_PARSER_H */ diff --git a/runtime/prop.c b/runtime/prop.c new file mode 100644 index 00000000..94d1bd49 --- /dev/null +++ b/runtime/prop.c @@ -0,0 +1,247 @@ +/* prop.c - rsyslog's prop object + * + * This object is meant to support message properties that are stored + * seperately from the message. The main intent is to support properties + * that are "constant" during a period of time, so that many messages may + * contain a reference to the same property. It is important, though, that + * properties are destroyed when they are no longer needed. + * + * Please note that this is a performance-critical part of the software and + * as such we may use some methods in here which do not look elegant, but + * which are fast... + * + * Module begun 2009-06-17 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <assert.h> +#include <string.h> + +#include "rsyslog.h" +#include "obj.h" +#include "obj-types.h" +#include "unicode-helper.h" +#include "atomic.h" +#include "prop.h" + +/* static data */ +DEFobjStaticHelpers + + +/* Standard-Constructor + */ +BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! */ + pThis->iRefCount = 1; +ENDobjConstruct(prop) + + +/* destructor for the prop object */ +BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ + int currRefCount; +CODESTARTobjDestruct(prop) + currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); + if(currRefCount == 0) { + /* (only) in this case we need to actually destruct the object */ + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + } else { + pThis = NULL; /* tell framework NOT to destructing the object! */ + } +ENDobjDestruct(prop) + +/* set string, we make our own private copy! This MUST only be called BEFORE + * ConstructFinalize()! + */ +static rsRetVal SetString(prop_t *pThis, uchar *psz, int len) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + pThis->len = len; + if(len < CONF_PROP_BUFSIZE) { + memcpy(pThis->szVal.sz, psz, len + 1); + } else { + CHKmalloc(pThis->szVal.psz = MALLOC(len + 1)); + memcpy(pThis->szVal.psz, psz, len + 1); + } + +finalize_it: + RETiRet; +} + + +/* get string length */ +static int GetStringLen(prop_t *pThis) +{ + return pThis->len; +} + + +/* get string */ +static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) +{ + BEGINfunc + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len < CONF_PROP_BUFSIZE) { + *ppsz = pThis->szVal.sz; + } else { + *ppsz = pThis->szVal.psz; + } + *plen = pThis->len; + ENDfunc + return RS_RET_OK; +} + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +propConstructFinalize(prop_t __attribute__((unused)) *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + RETiRet; +} + + +/* add a new reference. It is VERY IMPORTANT to call this function whenever + * the property is handed over to some entitiy that later call Destruct() on it. + */ +static rsRetVal AddRef(prop_t *pThis) +{ + ATOMIC_INC(pThis->iRefCount); + return RS_RET_OK; +} + + +/* this is a "do it all in one shot" function that creates a new property, + * assigns the provided string to it and finalizes the property. Among the + * convenience, it is alos (very, very) slightly faster. + * rgerhards, 2009-07-01 + */ +static rsRetVal CreateStringProp(prop_t **ppThis, uchar* psz, int len) +{ + DEFiRet; + propConstruct(ppThis); + SetString(*ppThis, psz, len); + propConstructFinalize(*ppThis); + RETiRet; +} + +/* another one-stop function, quite useful: it takes a property pointer and + * a string. If the string is already contained in the property, nothing happens. + * If the string is different (or the pointer NULL), the current property + * is destructed and a new one created. This can be used to get a specific + * name in those cases where there is a good chance that the property + * immediatly previously processed already contained the value we need - in + * which case we save us all the creation overhead by just reusing the already + * existing property). + * rgerhards, 2009-07-01 + */ +rsRetVal CreateOrReuseStringProp(prop_t **ppThis, uchar *psz, int len) +{ + uchar *pszPrev; + int lenPrev; + DEFiRet; + assert(ppThis != NULL); + + if(*ppThis == NULL) { + /* we need to create a property */ + CHKiRet(CreateStringProp(ppThis, psz, len)); + } else { + /* already exists, check if we can re-use it */ + GetString(*ppThis, &pszPrev, &lenPrev); + if(len != lenPrev || ustrcmp(psz, pszPrev)) { + /* different, need to discard old & create new one */ + propDestruct(ppThis); + CHKiRet(CreateStringProp(ppThis, psz, len)); + } /* else we can re-use the existing one! */ + } + +finalize_it: + RETiRet; +} + + +/* debugprint for the prop object */ +BEGINobjDebugPrint(prop) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(prop) + dbgprintf("prop object %p - no further debug info implemented\n", pThis); +ENDobjDebugPrint(prop) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(prop) +CODESTARTobjQueryInterface(prop) + if(pIf->ifVersion != propCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = propConstruct; + pIf->ConstructFinalize = propConstructFinalize; + pIf->Destruct = propDestruct; + pIf->DebugPrint = propDebugPrint; + pIf->SetString = SetString; + pIf->GetString = GetString; + pIf->GetStringLen = GetStringLen; + pIf->AddRef = AddRef; + pIf->CreateStringProp = CreateStringProp; + pIf->CreateOrReuseStringProp = CreateOrReuseStringProp; + +finalize_it: +ENDobjQueryInterface(prop) + + +/* Exit the prop class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(prop, OBJ_IS_CORE_MODULE) /* class, version */ +// objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(prop) + + +/* Initialize the prop class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(prop, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ +// CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, propDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, propConstructFinalize); +ENDObjClassInit(prop) + +/* vi:set ai: + */ diff --git a/runtime/prop.h b/runtime/prop.h new file mode 100644 index 00000000..e3519664 --- /dev/null +++ b/runtime/prop.h @@ -0,0 +1,58 @@ +/* The prop object. + * + * This implements props within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_PROP_H +#define INCLUDED_PROP_H + +/* the prop object */ +struct prop_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int iRefCount; /* reference counter */ + union { + uchar *psz; /* stored string */ + uchar sz[CONF_PROP_BUFSIZE]; + } szVal; + int len; /* we use int intentionally, otherwise we may get some troubles... */ +}; + +/* interfaces */ +BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(prop); + rsRetVal (*Construct)(prop_t **ppThis); + rsRetVal (*ConstructFinalize)(prop_t *pThis); + rsRetVal (*Destruct)(prop_t **ppThis); + rsRetVal (*SetString)(prop_t *pThis, uchar* psz, int len); + rsRetVal (*GetString)(prop_t *pThis, uchar** ppsz, int *plen); + int (*GetStringLen)(prop_t *pThis); + rsRetVal (*AddRef)(prop_t *pThis); + rsRetVal (*CreateStringProp)(prop_t **ppThis, uchar* psz, int len); + rsRetVal (*CreateOrReuseStringProp)(prop_t **ppThis, uchar *psz, int len); +ENDinterface(prop) +#define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(prop); + +#endif /* #ifndef INCLUDED_PROP_H */ diff --git a/runtime/queue.c b/runtime/queue.c index 4e017e84..b29ec7ac 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -8,7 +8,11 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * NOTE: as of 2009-04-22, I have begin to remove the qqueue* prefix from static + * function names - this makes it really hard to read and does not provide much + * benefit, at least I (now) think so... + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -49,71 +53,175 @@ #include "obj.h" #include "wtp.h" #include "wti.h" +#include "msg.h" #include "atomic.h" +#include "errmsg.h" +#include "datetime.h" +#include "unicode-helper.h" +#include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS # include <sched.h> -# define pthread_yield() sched_yield() #endif /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) +DEFobjCurrIf(strm) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) /* forward-definitions */ -rsRetVal qqueueChkPersist(qqueue_t *pThis); -static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); -static rsRetVal qqueueRateLimiter(qqueue_t *pThis); +static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates); +static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); -static int qqueueIsIdleDA(qqueue_t *pThis); -static rsRetVal qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); -static rsRetVal qqueueConsumerCancelCleanup(void *arg1, void *arg2); -static rsRetVal qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); +static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); +static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti); +static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 #define QUEUE_NO_CHECKPOINT 0 +/*********************************************************************** + * we need a private data structure, the "to-delete" list. As C does + * not provide any partly private data structures, we implement this + * structure right here inside the module. + * Note that this list must always be kept sorted based on a unique + * dequeue ID (which is monotonically increasing). + * rgerhards, 2009-05-18 + ***********************************************************************/ + +/* generate next uniqueue dequeue ID. Note that uniqueness is only required + * on a per-queue basis and while this instance runs. So a stricly monotonically + * increasing counter is sufficient (if enough bits are used). + */ +static inline qDeqID getNextDeqID(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->deqIDAdd++; +} + + +/* return the top element of the to-delete list or NULL, if the + * list is empty. + */ +static inline toDeleteLst_t *tdlPeek(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->toDeleteLst; +} + + +/* remove the top element of the to-delete list. Nothing but the + * element itself is destroyed. Must not be called when the list + * is empty. + */ +static inline rsRetVal tdlPop(qqueue_t *pQueue) +{ + toDeleteLst_t *pRemove; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + pRemove = pQueue->toDeleteLst; + pQueue->toDeleteLst = pQueue->toDeleteLst->pNext; + free(pRemove); + + RETiRet; +} + + +/* Add a new to-delete list entry. The function allocates the data + * structure, populates it with the values provided and links the new + * element into the correct place inside the list. + */ +static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElemDeq) +{ + toDeleteLst_t *pNew; + toDeleteLst_t *pPrev; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + CHKmalloc(pNew = MALLOC(sizeof(toDeleteLst_t))); + pNew->deqID = deqID; + pNew->nElemDeq = nElemDeq; + + /* now find right spot */ + for( pPrev = pQueue->toDeleteLst + ; pPrev != NULL && deqID > pPrev->deqID + ; pPrev = pPrev->pNext) { + /*JUST SEARCH*/; + } + + if(pPrev == NULL) { + pNew->pNext = pQueue->toDeleteLst; + pQueue->toDeleteLst = pNew; + } else { + pNew->pNext = pPrev->pNext; + pPrev->pNext = pNew; + } + +finalize_it: + RETiRet; +} + + /* methods */ -/* get the overall queue size, which includes ungotten objects. Must only be called +/* get the physical queue size. Must only be called * while mutex is locked! * rgerhards, 2008-01-29 */ static inline int -qqueueGetOverallQueueSize(qqueue_t *pThis) +getPhysicalQueueSize(qqueue_t *pThis) { -#if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */ -BEGINfunc -dbgoprint((obj_t*) pThis, "queue size: %d (regular %d, ungotten %d)\n", - pThis->iQueueSize + pThis->iUngottenObjs, pThis->iQueueSize, pThis->iUngottenObjs); -ENDfunc -#endif - return pThis->iQueueSize + pThis->iUngottenObjs; + return pThis->iQueueSize; +} + + +/* get the logical queue size (that is store size minus logically dequeued elements). + * Must only be called while mutex is locked! + * rgerhards, 2009-05-19 + */ +static inline int +getLogicalQueueSize(qqueue_t *pThis) +{ + return pThis->iQueueSize - pThis->nLogDeq; } + /* This function drains the queue in cases where this needs to be done. The most probable * reason is a HUP which needs to discard data (because the queue is configured to be lossy). * During a shutdown, this is typically not needed, as the OS frees up ressources and does * this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21 * This function returns void, as it makes no sense to communicate an error back, even if * it happens. + * This functions works "around" the regular deque mechanism, because it is only used to + * clean up (in cases where message loss is acceptable). */ static inline void queueDrain(qqueue_t *pThis) { void *pUsr; - ASSERT(pThis != NULL); + BEGINfunc + DBGOPRINT((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ - while(pThis->iQueueSize-- > 0) { - pThis->qDel(pThis, &pUsr); + while(ATOMIC_DEC_AND_FETCH(pThis->iQueueSize) > 0) { + pThis->qDeq(pThis, &pUsr); if(pUsr != NULL) { objDestruct(pUsr); } + pThis->qDel(pThis); } + ENDfunc } @@ -124,7 +232,8 @@ static inline void queueDrain(qqueue_t *pThis) * this point in time. The mutex must be locked when * ths function is called. -- rgerhards, 2008-01-25 */ -static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) +static inline rsRetVal +qqueueAdviseMaxWorkers(qqueue_t *pThis) { DEFiRet; int iMaxWorkers; @@ -132,96 +241,17 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); if(!pThis->bEnqOnly) { - if(pThis->bRunsDA) { - /* if we have not yet reached the high water mark, there is no need to start a - * worker. -- rgerhards, 2008-01-26 - */ - if(qqueueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ - } + if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) { + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } else { - if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { + if(getLogicalQueueSize(pThis) == 0) { + iMaxWorkers = 0; + } else if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { iMaxWorkers = 1; } else { - iMaxWorkers = qqueueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } - wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ - } - } - - RETiRet; -} - - -/* wait until we have a fully initialized DA queue. Sometimes, we need to - * sync with it, as we expect it for some function. - * rgerhards, 2008-02-27 - */ -static rsRetVal -qqueueWaitDAModeInitialized(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - - while(pThis->bRunsDA != 2) { - d_pthread_cond_wait(&pThis->condDAReady, pThis->mut); - } - - RETiRet; -} - - -/* Destruct DA queue. This is the last part of DA-to-normal-mode - * transistion. This is called asynchronously and some time quite a - * while after the actual transistion. The key point is that we need to - * do it at some later time, because we need to destruct the DA queue. That, - * however, can not be done in a thread that has been signalled - * This is to be called when we revert back to our own queue. - * This function must be called with the queue mutex locked (the wti - * class ensures this). - * rgerhards, 2008-01-15 - */ -static rsRetVal -qqueueTurnOffDAMode(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - - /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need - * to wait for its startup... -- rgerhards, 2008-01-25 - */ - qqueueWaitDAModeInitialized(pThis); - - /* if we need to pull any data that we still need from the (child) disk queue, - * now would be the time to do so. At present, we do not need this, but I'd like to - * keep that comment if future need arises. - */ - - /* we need to check if the DA queue is empty because the DA worker may simply have - * terminated do to no new messages arriving. That does not, however, mean that the - * DA queue is empty. If there is still data in that queue, we do nothing and leave - * that for a later incarnation of this function (it will be called multiple times - * during the lifetime of DA-mode, depending on how often the DA worker receives an - * inactivity timeout. -- rgerhards, 2008-01-25 - */ - if(pThis->pqDA->iQueueSize == 0) { - pThis->bRunsDA = 0; /* tell the world we are back in non-DA mode */ - /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, - * this will be quick. - */ - qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ - dbgoprint((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n", - iRet); - /* now we need to check if the regular queue has some messages. This may be the case - * when it is waiting that the high water mark is reached again. If so, we need to start up - * a regular worker. -- rgerhards, 2008-01-26 - */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - qqueueAdviseMaxWorkers(pThis); + wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); } } @@ -244,37 +274,26 @@ qqueueChkIsDA(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pszFilePrefix != NULL) { pThis->bIsDA = 1; - dbgoprint((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); + DBGOPRINT((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); } else { - dbgoprint((obj_t*) pThis, "is NOT disk-assisted\n"); + DBGOPRINT((obj_t*) pThis, "is NOT disk-assisted\n"); } RETiRet; } -/* Start disk-assisted queue mode. All internal settings are changed. This is supposed - * to be called from the DA worker, which must have been started before. The most important - * chore of this function is to create the DA queue object. If that function fails, - * the DA worker should return with an appropriate state, which in turn should lead to - * a re-set to non-DA mode in the Enq process. The queue mutex must be locked when this - * function is called, else a number of races will happen. - * Please note that this function may be called *while* we in DA mode. This is due to the - * fact that the DA worker calls it and the DA worker may be suspended (and restarted) due - * to inactivity timeouts. +/* Start disk-assisted queue mode. * rgerhards, 2008-01-15 */ static rsRetVal -qqueueStartDA(qqueue_t *pThis) +StartDA(qqueue_t *pThis) { DEFiRet; uchar pszDAQName[128]; ISOBJ_TYPE_assert(pThis, qqueue); - if(pThis->bRunsDA == 2) /* check if already in (fully initialized) DA mode... */ - FINALIZE; /* ... then we are already done! */ - /* create message queue */ CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); @@ -293,41 +312,25 @@ qqueueStartDA(qqueue_t *pThis) CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); + CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); - CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, pThis->toQShutdown)); CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0)); - if(pThis->toQShutdown == 0) { - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ - } else { - /* we use the shortest possible shutdown (0 is endless!) because when we run on disk AND - * have an obviously large backlog, we can't finish it in any case. So there is no point - * in holding shutdown longer than necessary. -- rgerhards, 2008-01-15 - */ - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 1)); - } iRet = qqueueStart(pThis->pqDA); /* file not found is expected, that means it is no previous QIF available */ - if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) + if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) { + errno = 0; /* else an errno is shown in errmsg! */ + errmsg.LogError(errno, iRet, "error starting up disk queue, using pure in-memory mode"); + pThis->bIsDA = 0; /* disable memory mode */ FINALIZE; /* something is wrong */ + } - /* as we are right now starting DA mode because we are so busy, it is - * extremely unlikely that any regular worker is sleeping on empty queue. HOWEVER, - * we want to be on the safe side, and so we awake anyone that is waiting - * on one. So even if the scheduler plays badly with us, things should be - * quite well. -- rgerhards, 2008-01-15 - */ - wtpWakeupWrkr(pThis->pWtpReg); /* awake all workers, but not ourselves ;) */ - - pThis->bRunsDA = 2; /* we are now in DA mode, but not fully initialized */ - pThis->bChildIsDone = 0;/* set to 1 when child's worker detect queue is finished */ - pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */ - - dbgoprint((obj_t*) pThis, "is now running in disk assisted mode, disk queue 0x%lx\n", + DBGOPRINT((obj_t*) pThis, "DA queue initialized, disk queue 0x%lx\n", qqueueGetID(pThis->pqDA)); finalize_it: @@ -335,7 +338,7 @@ finalize_it: if(pThis->pqDA != NULL) { qqueueDestruct(&pThis->pqDA); } - dbgoprint((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); + DBGOPRINT((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); pThis->bIsDA = 0; } @@ -349,8 +352,8 @@ finalize_it: * If this function fails (should not happen), DA mode is not turned on. * rgerhards, 2008-01-16 */ -static inline rsRetVal -qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +static rsRetVal +InitDA(qqueue_t *pThis, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -363,82 +366,30 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) * is intentional. We assume that when we need it once, we may also need it on another * occasion. Ressources used are quite minimal when no worker is running. * rgerhards, 2008-01-24 + * NOTE: this is the DA worker *pool*, not the DA queue! */ - if(pThis->pWtpDA == NULL) { - lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DA", obj.GetName((obj_t*) pThis)); - CHKiRet(wtpConstruct (&pThis->pWtpDA)); - CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerDA)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) qqueueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); - CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); - CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); - CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); - CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); - CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); - CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); - } + lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis)); + CHKiRet(wtpConstruct (&pThis->pWtpDA)); + CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerDA)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); + CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); + CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); + CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); + CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); + CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); + CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); /* if we reach this point, we have a "good" DA worker pool */ - /* indicate we now run in DA mode - this is reset by the DA worker if it fails */ - pThis->bRunsDA = 1; - pThis->bDAEnqOnly = bEnqOnly; - - /* now we must now adivse the wtp that we need one worker. If none is yet active, - * that will also start one up. If we forgot that step, everything would be stalled - * until the next enqueue request. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues alsways have just one worker max */ - -finalize_it: - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - RETiRet; -} - - -/* check if we need to start disk assisted mode and send some signals to - * keep it running if we are already in it. It also checks if DA mode is - * partially initialized, in which case it waits for initialization to - * complete. - * rgerhards, 2008-01-14 - */ -static inline rsRetVal -qqueueChkStrtDA(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - /* if we do not hit the high water mark, we have nothing to do */ - if(qqueueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) - ABORT_FINALIZE(RS_RET_OK); - - if(pThis->bRunsDA) { - /* then we need to signal that we are at the high water mark again. If that happens - * on our way down the queue, that doesn't matter, because then nobody is waiting - * on the condition variable. - * (Remember that a DA queue stops draining the queue once it has reached the low - * water mark and restarts it when the high water mark is reached again - this is - * what this code here is responsible for. Please note that all workers may have been - * terminated due to the inactivity timeout, thus we need to advise the pool that - * we need at least one). - */ - dbgoprint((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n", - qqueueGetOverallQueueSize(pThis)); - qqueueAdviseMaxWorkers(pThis); - } else { - /* this is the case when we are currently not running in DA mode. So it is time - * to turn it back on. - */ - dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", - qqueueGetOverallQueueSize(pThis)); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ + /* now construct the actual queue (if it does not already exist) */ + if(pThis->pqDA == NULL) { + CHKiRet(StartDA(pThis)); } finalize_it: + END_MTX_PROTECTED_OPERATIONS(pThis->mut); RETiRet; } @@ -462,10 +413,11 @@ static rsRetVal qConstructFixedArray(qqueue_t *pThis) if(pThis->iMaxQueueSize == 0) ABORT_FINALIZE(RS_RET_QSIZE_ZERO); - if((pThis->tVars.farray.pBuf = malloc(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { + if((pThis->tVars.farray.pBuf = MALLOC(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } + pThis->tVars.farray.deqhead = 0; pThis->tVars.farray.head = 0; pThis->tVars.farray.tail = 0; @@ -483,9 +435,7 @@ static rsRetVal qDestructFixedArray(qqueue_t *pThis) ASSERT(pThis != NULL); queueDrain(pThis); /* discard any remaining queue entries */ - - if(pThis->tVars.farray.pBuf != NULL) - free(pThis->tVars.farray.pBuf); + free(pThis->tVars.farray.pBuf); RETiRet; } @@ -504,76 +454,37 @@ static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) RETiRet; } -static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out) + +static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out) { DEFiRet; ASSERT(pThis != NULL); - *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.head]; + *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.deqhead]; - pThis->tVars.farray.head++; - if (pThis->tVars.farray.head == pThis->iMaxQueueSize) - pThis->tVars.farray.head = 0; + pThis->tVars.farray.deqhead++; + if (pThis->tVars.farray.deqhead == pThis->iMaxQueueSize) + pThis->tVars.farray.deqhead = 0; RETiRet; } -/* -------------------- linked list -------------------- */ - -/* first some generic functions which are also used for the unget linked list */ - -static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) +static rsRetVal qDelFixedArray(qqueue_t *pThis) { DEFiRet; - qLinkedList_t *pEntry; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - pEntry->pNext = NULL; - pEntry->pUsr = pUsr; + ASSERT(pThis != NULL); - if(*ppRoot == NULL) { - *ppRoot = *ppLast = pEntry; - } else { - (*ppLast)->pNext = pEntry; - *ppLast = pEntry; - } + pThis->tVars.farray.head++; + if (pThis->tVars.farray.head == pThis->iMaxQueueSize) + pThis->tVars.farray.head = 0; -finalize_it: RETiRet; } -static inline rsRetVal qqueueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) -{ - DEFiRet; - qLinkedList_t *pEntry; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - ASSERT(ppUsr != NULL); - ASSERT(*ppRoot != NULL); - - pEntry = *ppRoot; - *ppUsr = pEntry->pUsr; - - if(*ppRoot == *ppLast) { - *ppRoot = NULL; - *ppLast = NULL; - } else { - *ppRoot = pEntry->pNext; - } - free(pEntry); - - RETiRet; -} -/* end generic functions which are also used for the unget linked list */ +/* -------------------- linked list -------------------- */ static rsRetVal qConstructLinkedList(qqueue_t *pThis) @@ -582,8 +493,9 @@ static rsRetVal qConstructLinkedList(qqueue_t *pThis) ASSERT(pThis != NULL); - pThis->tVars.linklist.pRoot = 0; - pThis->tVars.linklist.pLast = 0; + pThis->tVars.linklist.pDeqRoot = NULL; + pThis->tVars.linklist.pDelRoot = NULL; + pThis->tVars.linklist.pLast = NULL; qqueueChkIsDA(pThis); @@ -606,54 +518,59 @@ static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) { + qLinkedList_t *pEntry; DEFiRet; - iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); -#if 0 - qLinkedList_t *pEntry; - - ASSERT(pThis != NULL); - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc((pEntry = (qLinkedList_t*) MALLOC(sizeof(qLinkedList_t)))); pEntry->pNext = NULL; pEntry->pUsr = pUsr; - if(pThis->tVars.linklist.pRoot == NULL) { - pThis->tVars.linklist.pRoot = pThis->tVars.linklist.pLast = pEntry; + if(pThis->tVars.linklist.pDelRoot == NULL) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = pEntry; } else { pThis->tVars.linklist.pLast->pNext = pEntry; pThis->tVars.linklist.pLast = pEntry; } + if(pThis->tVars.linklist.pDeqRoot == NULL) { + pThis->tVars.linklist.pDeqRoot = pEntry; + } + finalize_it: -#endif RETiRet; } -static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr) + +static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr) { - DEFiRet; - iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); -#if 0 qLinkedList_t *pEntry; + DEFiRet; - ASSERT(pThis != NULL); - ASSERT(pThis->tVars.linklist.pRoot != NULL); - - pEntry = pThis->tVars.linklist.pRoot; + pEntry = pThis->tVars.linklist.pDeqRoot; + ISOBJ_TYPE_assert(pEntry->pUsr, msg); *ppUsr = pEntry->pUsr; + pThis->tVars.linklist.pDeqRoot = pEntry->pNext; - if(pThis->tVars.linklist.pRoot == pThis->tVars.linklist.pLast) { - pThis->tVars.linklist.pRoot = NULL; - pThis->tVars.linklist.pLast = NULL; + RETiRet; +} + + +static rsRetVal qDelLinkedList(qqueue_t *pThis) +{ + qLinkedList_t *pEntry; + DEFiRet; + + pEntry = pThis->tVars.linklist.pDelRoot; + + if(pThis->tVars.linklist.pDelRoot == pThis->tVars.linklist.pLast) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = NULL; } else { - pThis->tVars.linklist.pRoot = pEntry->pNext; + pThis->tVars.linklist.pDelRoot = pEntry->pNext; } + free(pEntry); -#endif RETiRet; } @@ -667,45 +584,7 @@ qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pTh DEFiRet; ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_TYPE_assert(pThis, qqueue); - CHKiRet(strmSetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); -finalize_it: - RETiRet; -} - - -/* This method checks if we have a QIF file for the current queue (no matter of - * queue mode). Returns RS_RET_OK if we have a QIF file or an error status otherwise. - * rgerhards, 2008-01-15 - */ -static rsRetVal -qqueueHaveQIF(qqueue_t *pThis) -{ - DEFiRet; - uchar pszQIFNam[MAXFNAME]; - size_t lenQIFNam; - struct stat stat_buf; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pszFilePrefix == NULL) - ABORT_FINALIZE(RS_RET_NO_FILEPREFIX); - - /* Construct file name */ - lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", - (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - - /* check if the file exists */ - if(stat((char*) pszQIFNam, &stat_buf) == -1) { - if(errno == ENOENT) { - dbgoprint((obj_t*) pThis, "no .qi file found\n"); - ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); - } else { - dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno); - ABORT_FINALIZE(RS_RET_IO_ERROR); - } - } - /* If we reach this point, we have a .qi file */ - + CHKiRet(strm.SetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); finalize_it: RETiRet; } @@ -722,8 +601,6 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; struct stat stat_buf; - int iUngottenObjs; - obj_t *pUsr; ISOBJ_TYPE_assert(pThis, qqueue); @@ -734,44 +611,41 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* check if the file exists */ if(stat((char*) pszQIFNam, &stat_buf) == -1) { if(errno == ENOENT) { - dbgoprint((obj_t*) pThis, "clean startup, no .qi file found\n"); + DBGOPRINT((obj_t*) pThis, "clean startup, no .qi file found\n"); ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); } else { - dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno); + DBGOPRINT((obj_t*) pThis, "error %d trying to access .qi file\n", errno); ABORT_FINALIZE(RS_RET_IO_ERROR); } } /* If we reach this point, we have a .qi file */ - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_READ)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_READ)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, we try to read the property bag for ourselfs */ CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF)); - /* then the ungotten object queue */ - iUngottenObjs = pThis->iUngottenObjs; - pThis->iUngottenObjs = 0; /* will be incremented when we add objects! */ - - while(iUngottenObjs > 0) { - /* fill the queue from disk */ - CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL)); - qqueueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); - --iUngottenObjs; /* one less */ - } - - /* and now the stream objects (some order as when persisted!) */ + /* then the stream objects (same order as when persisted!) */ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, + CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pRead)); + /* create a duplicate for the read "pointer". + */ + + CHKiRet(strm.Dup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */ + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); + + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDel)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDeq)); /* OK, we could successfully read the file, so we now can request that it be * deleted when we are done with the persisted information. @@ -780,10 +654,10 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); if(iRet != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", + DBGOPRINT((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", iRet); } @@ -815,24 +689,34 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) if(bRestarted == 1) { ; } else { - CHKiRet(strmConstruct(&pThis->tVars.disk.pWrite)); - CHKiRet(strmSetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); - CHKiRet(strmSetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pWrite)); - - CHKiRet(strmConstruct(&pThis->tVars.disk.pRead)); - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); - CHKiRet(strmSetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pRead)); - - - CHKiRet(strmSetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(strmSetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pWrite, pThis->bSyncQueueFiles)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); + + CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); + + CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDel)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pReadDel, pThis->bSyncQueueFiles)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDel, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDel)); + + CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDeq, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDel, pThis->pszFilePrefix, pThis->lenFilePrefix)); } /* now we set (and overwrite in case of a persisted restart) some parameters which @@ -840,8 +724,9 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) * for example file name generation must not be changed as that would break the * ability to read existing queue files. -- rgerhards, 2008-01-12 */ - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDeq, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDel, pThis->iMaxFileSize)); finalize_it: RETiRet; @@ -854,8 +739,12 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) ASSERT(pThis != NULL); - strmDestruct(&pThis->tVars.disk.pWrite); - strmDestruct(&pThis->tVars.disk.pRead); + if(pThis->tVars.disk.pWrite != NULL) + strm.Destruct(&pThis->tVars.disk.pWrite); + if(pThis->tVars.disk.pReadDeq != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDeq); + if(pThis->tVars.disk.pReadDel != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDel); RETiRet; } @@ -867,10 +756,10 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) ASSERT(pThis != NULL); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); CHKiRet((objSerialize(pUsr))(pUsr, pThis->tVars.disk.pWrite)); - CHKiRet(strmFlush(pThis->tVars.disk.pWrite)); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ + CHKiRet(strm.Flush(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ pThis->tVars.disk.sizeOnDisk += nWriteCount; @@ -880,23 +769,37 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) */ objDestruct(pUsr); - dbgoprint((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets\n", + DBGOPRINT((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets\n", nWriteCount, pThis->tVars.disk.sizeOnDisk); finalize_it: RETiRet; } -static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) + +static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr) +{ + DEFiRet; + + CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +static rsRetVal qDelDisk(qqueue_t *pThis) { + obj_t *pDummyObj; /* we need to deserialize it... */ DEFiRet; int64 offsIn; int64 offsOut; - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); - CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pRead, NULL, NULL)); - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsIn)); + CHKiRet(obj.Deserialize(&pDummyObj, (uchar*) "msg", pThis->tVars.disk.pReadDel, NULL, NULL)); + objDestruct(pDummyObj); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsOut)); /* This time it is a bit tricky: we free disk space only upon file deletion. So we need * to keep track of what we have read until we get an out-offset that is lower than the @@ -908,7 +811,7 @@ static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) } else { pThis->tVars.disk.sizeOnDisk -= pThis->tVars.disk.bytesRead; pThis->tVars.disk.bytesRead = offsOut; - dbgoprint((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk); + DBGOPRINT((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk); /* awake possibly waiting enq process */ pthread_cond_signal(&pThis->notFull); /* we hold the mutex while we are in here! */ } @@ -917,6 +820,7 @@ finalize_it: RETiRet; } + /* -------------------- direct (no queueing) -------------------- */ static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { @@ -931,6 +835,8 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { + batch_t singleBatch; + batch_obj_t batchObj; DEFiRet; ASSERT(pThis != NULL); @@ -940,13 +846,21 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) * mode the consumer probably has a lot to convey (which get's lost in the other modes * because they are asynchronous. But direct mode is deliberately synchronous. * rgerhards, 2008-02-12 + * We use our knowledge about the batch_t structure below, but without that, we + * pay a too-large performance toll... -- rgerhards, 2009-04-22 */ - iRet = pThis->pConsumer(pThis->pUsr, pUsr); + batchObj.state = BATCH_STATE_RDY; + batchObj.pUsrp = (obj_t*) pUsr; + singleBatch.nElem = 1; /* there always is only one in direct mode */ + singleBatch.pElem = &batchObj; + iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate); + objDestruct(pUsr); RETiRet; } -static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) + +static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) { return RS_RET_OK; } @@ -955,57 +869,6 @@ static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute_ /* --------------- end type-specific handlers -------------------- */ -/* unget a user pointer that has been dequeued. This functionality is especially important - * for consumer cancel cleanup handlers. To support it, a short list of ungotten user pointers - * is maintened in memory. - * rgerhards, 2008-01-20 - */ -static rsRetVal -qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) -{ - DEFiRet; - DEFVARS_mutexProtection; - - ISOBJ_TYPE_assert(pThis, qqueue); - ISOBJ_assert(pUsr); /* TODO: we aborted right at this place at least 3 times -- race? 2008-02-28, -03-10, -03-15 - The second time I noticed it the queue was in destruction with NO worker threads - running. The pUsr ptr was totally off and provided no clue what it may be pointing - at (except that it looked like the static data pool). Both times, the abort happend - inside an action queue */ - - dbgoprint((obj_t*) pThis, "ungetting user object %s\n", obj.GetName(pUsr)); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); - iRet = qqueueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); - ++pThis->iUngottenObjs; /* indicate one more */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - - RETiRet; -} - - -/* dequeues a user pointer from the ungotten queue. Pointers from there should always be - * dequeued first. - * - * This function must only be called when the mutex is locked! - * - * rgerhards, 2008-01-29 - */ -static rsRetVal -qqueueGetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(ppUsr != NULL); - - iRet = qqueueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); - --pThis->iUngottenObjs; /* indicate one less */ - dbgoprint((obj_t*) pThis, "dequeued ungotten user object %s\n", obj.GetName(*ppUsr)); - - RETiRet; -} - - /* generic code to add a queue entry * We use some specific code to most efficiently support direct mode * queues. This is justified in spite of the gain and the need to do some @@ -1022,7 +885,8 @@ qqueueAdd(qqueue_t *pThis, void *pUsr) if(pThis->qType != QUEUETYPE_DIRECT) { ATOMIC_INC(pThis->iQueueSize); - dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize); + DBGOPRINT((obj_t*) pThis, "entry added, size now log %d, phys %d entries\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); } finalize_it: @@ -1030,12 +894,10 @@ finalize_it: } -/* generic code to remove a queue entry - * rgerhards, 2008-01-29: we must first see if there is any object in the - * ungotten list and, if so, dequeue it first. +/* generic code to dequeue a queue entry */ static rsRetVal -qqueueDel(qqueue_t *pThis, void *pUsr) +qqueueDeq(qqueue_t *pThis, void **ppUsr) { DEFiRet; @@ -1046,224 +908,247 @@ qqueueDel(qqueue_t *pThis, void *pUsr) * If we decrement, however, we may lose a message. But that is better than * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ - if(pThis->iUngottenObjs > 0) { - iRet = qqueueGetUngottenObj(pThis, (obj_t**) pUsr); - } else { - iRet = pThis->qDel(pThis, pUsr); - ATOMIC_DEC(pThis->iQueueSize); - } + iRet = pThis->qDeq(pThis, ppUsr); + ATOMIC_INC(pThis->nLogDeq); - dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n", - iRet, pThis->iQueueSize); +// DBGOPRINT((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", +// getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } -/* This function shuts down all worker threads and waits until they - * have terminated. If they timeout, they are cancelled. Parameters have been set - * before this function is called so that DA queues will be fully persisted to - * disk (if configured to do so). - * rgerhards, 2008-01-24 - * Please note that this function shuts down BOTH the parent AND the child queue - * in DA case. This is necessary because their timeouts are tightly coupled. Most - * importantly, the timeouts would be applied twice (or logic be extremely - * complex) if each would have its own shutdown. The function does not self check - * this condition - the caller must make sure it is not called with a parent. +/* Try to shut down regular and DA queue workers, within the queue timeout + * period. That means processing continues as usual. This is the expected + * usual case, where during shutdown those messages remaining are being + * processed. At this point, it is acceptable that the queue can not be + * fully depleted, that case is handled in the next step. During this phase, + * we first shut down the main queue DA worker to prevent new data to arrive + * at the DA queue, and then we ask the regular workers of both the Regular + * and DA queue to try complete processing. + * rgerhards, 2009-10-14 */ -static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) +static inline rsRetVal +tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; struct timespec tTimeout; rsRetVal iRetLocal; + DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ - dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); + if(pThis->bIsDA) { + /* We need to lock the mutex, as otherwise we may have a race that prevents + * us from awaking the DA worker. */ + d_pthread_mutex_lock(pThis->mut); - /* we reduce the low water mark in any case. This is not absolutely necessary, but - * it is useful because we enable DA mode at several spots below and so we do not need - * to think about the low water mark each time. - */ - pThis->iHighWtrMrk = 1; /* if we do not do this, the DA queue will not stop! */ - pThis->iLowWtrMrk = 0; + /* tell regular queue DA worker to stop shuffling messages to DA queue... */ + pThis->pqDA->bEnqOnly = 1; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE); + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); + DBGOPRINT((obj_t*) pThis, "awoke DA worker, told it to shut down.\n"); - /* first try to shutdown the queue within the regular shutdown period */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - if(pThis->bRunsDA) { - /* We may have waited on the low water mark. As it may have changed, we - * see if we reactivate the worker. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); - } + /* also tell the DA queue worker to shut down, so that it already knows... */ + wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN); + wtpAdviseMaxWorkers(pThis->pqDA->pWtpReg, 1); /* awake its lone worker */ + DBGOPRINT((obj_t*) pThis, "awoke DA queue regular worker, told it to shut down when done.\n"); + + d_pthread_mutex_unlock(pThis->mut); } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - /* Now wait for the queue's workers to shut down. Note that we run into the code even if we just found - * out there are no active workers - that doesn't matter: the wtp knows about that and so will - * return immediately. - * We do not yet care about the DA worker - that will be handled down later in the process. - * Note that we must not request shutdown right now - that may introduce a race: if the regular queue - * still runs DA assisted and the DA worker gets scheduled first, it will terminate itself (if the DA - * queue happens to be empty at that instant). Then the regular worker enqueues messages, what will lead - * to a restart of the worker. Of course, everything will continue to run, but in a bit sub-optimal way - * (from a performance point of view). So we don't do anything right now. The DA queue will continue to - * process messages and shutdown itself in any case if there is nothing to do. So we don't loose anything - * by not requesting shutdown now. - * rgerhards, 2008-01-25 - */ + /* first calculate absolute timeout - we need the absolute value here, because we need to coordinate * shutdown of both the regular and DA queue on *the same* timeout. */ timeoutComp(&tTimeout, pThis->toQShutdown); - dbgoprint((obj_t*) pThis, "trying shutdown of regular workers\n"); + DBGOPRINT((obj_t*) pThis, "trying shutdown of regular workers\n"); iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN, &tTimeout); if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n"); + DBGOPRINT((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n"); } else { - /* OK, the regular queue is now shut down. So we can now wait for the DA queue (if running DA) */ - dbgoprint((obj_t*) pThis, "regular queue workers shut down.\n"); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(pThis->bRunsDA) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", - qqueueGetID(pThis->pqDA)); - /* we use the same absolute timeout as above, so we do not use more than the configured - * timeout interval! - */ - dbgoprint((obj_t*) pThis, "trying shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "shutdown timed out on DA queue (this is OK)\n"); - } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - } + DBGOPRINT((obj_t*) pThis, "regular queue workers shut down.\n"); } - /* when we reach this point, both queues are either empty or the regular queue shutdown timeout - * has expired. Now we need to check if we are configured to not loose messages. If so, we need - * to persist the queue to disk (this is only possible if the queue is DA-enabled). We must also - * set the primary queue to SHUTDOWN_IMMEDIATE, as it shall now terminate as soon as its consumer - * is done. This is especially important as we otherwise may interfere with queue order while the - * DA consumer is running. -- rgerhards, 2008-01-27 - * Note: there was a note that we should not wait eternally on the DA worker if we run in - * enqueue-only note. I have reviewed the code and think there is no need for this check. Howerver, - * I'd like to keep this note in here should we happen to run into some related trouble. - * rgerhards, 2008-01-28 - */ - wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); /* set primary queue to shutdown only */ - - /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ - if(pThis->bRunsDA) - qqueueWaitDAModeInitialized(pThis); - - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - /* optimize parameters for shutdown of DA-enabled queues */ - if(pThis->bIsDA && qqueueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { - /* switch to enqueue-only mode so that no more actions happen */ - if(pThis->bRunsDA == 0) { - qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ + /* OK, the worker for the regular queue is processed, on the the DA queue regular worker. */ + if(pThis->pqDA != NULL) { + DBGOPRINT((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", + qqueueGetID(pThis->pqDA)); + /* we use the same absolute timeout as above, so we do not use more than the configured + * timeout interval! + */ + DBGOPRINT((obj_t*) pThis, "trying shutdown of regular worker of DA queue\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "shutdown timed out on DA queue worker (this is OK)\n"); } else { - /* TODO: RACE: we may reach this point when the DA worker has been initialized (state 1) - * but is not yet running (state 2). In this case, pThis->pqDA is NULL! rgerhards, 2008-02-27 - */ - qqueueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ + DBGOPRINT((obj_t*) pThis, "DA queue worker shut down.\n"); } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - /* make sure we do not timeout before we are done */ - dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, eternal timeout set\n"); - timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); - /* and run the primary queue's DA worker to drain the queue */ - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " - "continuing, but results are unpredictable\n", iRetLocal); - } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); } - /* now the primary queue is either empty, persisted to disk - or set to loose messages. So we - * can now request immediate shutdown of any remaining workers. Note that if bSaveOnShutdown was set, - * the queue is now empty. If regular workers are still running, and try to pull the next message, - * they will automatically terminate as there no longer is any message left to process. - */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - timeoutComp(&tTimeout, pThis->toActShutdown); - if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of regular workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } - /* we need to re-aquire the mutex for the next check in this case! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ + RETiRet; +} + + +/* Try to shut down regular and DA queue workers, within the action timeout + * period. This aborts processing, but at the end of the current action, in + * a well-defined manner. During this phase, we terminate all three worker + * pools, including the regular queue DA worker if it not yet has terminated. + * Not finishing processing all messages is OK (and expected) at this stage + * (they may be preserved later, depending * on bSaveOnShutdown setting). + * rgerhards, 2009-10-14 + */ +static rsRetVal +tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) +{ + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + +RUNLOG_STR("trying to shutdown workers within Action Timeout"); + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + /* instruct workers to finish ASAP, even if still work exists */ + pThis->bEnqOnly = 1; + pThis->bShutdownImmediate = 1; + /* now DA queue */ + if(pThis->bIsDA) { + pThis->pqDA->bEnqOnly = 1; + pThis->pqDA->bShutdownImmediate = 1; + } + +// TODO: make sure we have at minimum a 10ms timeout - workers deserve a chance... + /* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */ + timeoutComp(&tTimeout, pThis->toActShutdown); + DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of regular workers (if any)\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " + "triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " + "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); + } + + if(pThis->pqDA != NULL) { + /* and now the same for the DA queue */ + DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of DA queue workers\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable " + "and triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA " + "queue in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); } - if(pThis->bIsDA && wtpGetCurNumWrkr(pThis->pWtpDA, LOCK_MUTEX) > 0) { - /* and now the same for the DA queue */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } + + /* and now we need to terminate the DA worker itself. We always grant it a 100ms timeout, + * which should be sufficient and usually not be required (it is expected to have finished + * long before while we were processing the queue timeout in shutdown phase 1). + * rgerhards, 2009-10-14 + */ + timeoutComp(&tTimeout, 100); + DBGOPRINT((obj_t*) pThis, "trying regular shutdown of main queue DA worker pool\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "shutdown timed out on main queue DA worker pool " + "(this is not good, but probably OK)\n"); } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + DBGOPRINT((obj_t*) pThis, "main queue DA worker pool shut down.\n"); } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); } + RETiRet; +} + + +/* This function cancels all remaining regular workers for both the main and the DA + * queue. + * rgerhards, 2009-05-29 + */ +static rsRetVal +cancelWorkers(qqueue_t *pThis) +{ + rsRetVal iRetLocal; + DEFiRet; + /* Now queue workers should have terminated. If not, we need to cancel them as we have applied * all timeout setting. If any worker in any queue still executes, its consumer is possibly - * long-running and cancelling is the only way to get rid of it. Note that the - * cancellation handler will probably re-queue a user pointer, so the queue's enqueue - * function is still needed (what is no problem as we do not yet destroy the queue - but I - * thought it's a good idea to mention that fact). -- rgerhards, 2008-01-25 + * long-running and cancelling is the only way to get rid of it. */ - dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n"); + DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n"); iRetLocal = wtpCancelAll(pThis->pWtpReg); /* returns immediately if all threads already have terminated */ if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker " + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker " "threads, continuing, but results are unpredictable\n", iRetLocal); } - - /* TODO: think: do we really need to do this here? Can't it happen on DA queue destruction? If we - * disable it, we get an assertion... I think this is OK, as we need to have a certain order and - * canceling the DA workers here ensures that order. But in any instant, we may have a look at this - * code after we have reaced the milestone. -- rgerhards, 2008-01-27 - */ /* ... and now the DA queue, if it exists (should always be after the primary one) */ if(pThis->pqDA != NULL) { - dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n"); + DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n"); iRetLocal = wtpCancelAll(pThis->pqDA->pWtpReg); /* returns immediately if all threads already have terminated */ if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker " + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker " "threads, continuing, but results are unpredictable\n", iRetLocal); } + + /* finally, we cancel the main queue's DA worker pool, if it still is running. It may be + * restarted later to persist the queue. But we stop it, because otherwise we get into + * big trouble when resetting the logical dequeue pointer. This operation can only be + * done when *no* worker is running. So time for a shutdown... -- rgerhards, 2009-05-28 + */ + DBGOPRINT((obj_t*) pThis, "checking to see if main queue DA worker pool needs to be cancelled\n"); + iRetLocal = wtpCancelAll(pThis->pWtpDA); /* returns immediately if all threads already have terminated */ } + RETiRet; +} + + +/* This function shuts down all worker threads and waits until they + * have terminated. If they timeout, they are cancelled. + * rgerhards, 2008-01-24 + * Please note that this function shuts down BOTH the parent AND the child queue + * in DA case. This is necessary because their timeouts are tightly coupled. Most + * importantly, the timeouts would be applied twice (or logic be extremely + * complex) if each would have its own shutdown. The function does not self check + * this condition - the caller must make sure it is not called with a parent. + * rgerhards, 2009-05-26: we do NO longer persist the queue here if bSaveOnShutdown + * is set. This must be handled by the caller. Not doing that cleans up the queue + * shutdown considerably. Also, older engines had a potential hang condition when + * the DA queue was already started and the DA worker configured for infinite + * retries and the action was during retry processing. This was a design issue, + * which is solved as of now. Note that the shutdown now may take a little bit + * longer, because we no longer can persist the queue in parallel to waiting + * on worker timeouts. + */ +static rsRetVal +ShutdownWorkers(qqueue_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + DBGOPRINT((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); + + CHKiRet(tryShutdownWorkersWithinQueueTimeout(pThis)); + + if(getPhysicalQueueSize(pThis) > 0) { + CHKiRet(tryShutdownWorkersWithinActionTimeout(pThis)); + } + + CHKiRet(cancelWorkers(pThis)); + /* ... finally ... all worker threads have terminated :-) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers - * may still be running. + * may still be running. Note that the main queue's DA worker may still be running. */ - dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", qqueueGetOverallQueueSize(pThis)); + DBGOPRINT((obj_t*) pThis, "worker threads terminated, remaining queue size log %d, phys %d.\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); +finalize_it: RETiRet; } @@ -1275,7 +1160,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * to modify some parameters before the queue is actually started. */ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)) + int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*,int*)) { DEFiRet; qqueue_t *pThis; @@ -1284,13 +1169,10 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread ASSERT(pConsumer != NULL); ASSERT(iWorkerThreads >= 0); - if((pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))); /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -1298,13 +1180,15 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->iFullDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 3; /* default 97% */ pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */ - pThis->lenSpoolDir = strlen((char*)pThis->pszSpoolDir); + pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; + pThis->nLogDeq = 0; pThis->iMaxQueueSize = iMaxQueueSize; pThis->pConsumer = pConsumer; pThis->iNumWorkerThreads = iWorkerThreads; pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */ + pThis->iDeqBatchSize = 8; /* conservative default, should still provide good performance */ pThis->pszFilePrefix = NULL; pThis->qType = qType; @@ -1315,18 +1199,21 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qConstruct = qConstructFixedArray; pThis->qDestruct = qDestructFixedArray; pThis->qAdd = qAddFixedArray; + pThis->qDeq = qDeqFixedArray; pThis->qDel = qDelFixedArray; break; case QUEUETYPE_LINKEDLIST: pThis->qConstruct = qConstructLinkedList; pThis->qDestruct = qDestructLinkedList; pThis->qAdd = qAddLinkedList; - pThis->qDel = (rsRetVal (*)(qqueue_t*,void**)) qDelLinkedList; + pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList; + pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; pThis->qDestruct = qDestructDisk; pThis->qAdd = qAddDisk; + pThis->qDeq = qDeqDisk; pThis->qDel = qDelDisk; /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ @@ -1345,40 +1232,10 @@ finalize_it: } -/* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. - * Params: - * arg1 - user pointer (in this case a qqueue_t) - * arg2 - user data pointer (in this case a queue data element, any object [queue's pUsr ptr!]) - * Note that arg2 may be NULL, in which case no dequeued but unprocessed pUsr exists! - * rgerhards, 2008-01-16 - */ -static rsRetVal -qqueueConsumerCancelCleanup(void *arg1, void *arg2) -{ - DEFiRet; - - qqueue_t *pThis = (qqueue_t*) arg1; - obj_t *pUsr = (obj_t*) arg2; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pUsr != NULL) { - /* make sure the data element is not lost */ - dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n"); - CHKiRet(qqueueUngetObj(pThis, pUsr, LOCK_MUTEX)); - } - -finalize_it: - RETiRet; -} - - - /* This function checks if the provided message shall be discarded and does so, if needed. * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to * provide real-time creation of spool files. - * Note: cached copies of iQueueSize and bRunsDA are provided so that no mutex locks are required. + * Note: cached copies of iQueueSize is provided so that no mutex locks are required. * The caller must have obtained them while the mutex was locked. Of course, these values may no * longer be current, but that is OK for the discard check. At worst, the message is either processed * or discarded when it should not have been. As discarding is in itself somewhat racy and erratic, @@ -1388,7 +1245,7 @@ finalize_it: * the return state! * rgerhards, 2008-01-24 */ -static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) +static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, void *pUsr) { DEFiRet; rsRetVal iRetLocal; @@ -1397,15 +1254,15 @@ static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, voi ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); - if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk && bRunsDA == 0) { + if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk) { iRetLocal = objGetSeverity(pUsr, &iSeverity); if(iRetLocal == RS_RET_OK && iSeverity >= pThis->iDiscardSeverity) { - dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", + DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", iQueueSize, iSeverity); objDestruct(pUsr); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } else { - dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg " + DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg " "(iRet: %d, severity %d)\n", iQueueSize, iRetLocal, iSeverity); } } @@ -1415,78 +1272,207 @@ finalize_it: } -/* dequeue the queued object for the queue consumers. - * rgerhards, 2008-10-21 +/* Finally remove n elements from the queue store. */ -static rsRetVal -qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +static inline rsRetVal +DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) +{ + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + /* now send delete request to storage driver */ + for(i = 0 ; i < nElem ; ++i) { + pThis->qDel(pThis); + } + + /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ + ATOMIC_SUB(pThis->iQueueSize, nElem); + ATOMIC_SUB(pThis->nLogDeq, nElem); +dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + ++pThis->deqIDDel; /* one more batch dequeued */ + + RETiRet; +} + + +/* remove messages from the physical queue store that are fully processed. This is + * controlled via the to-delete list. + */ +static inline rsRetVal +DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) { + toDeleteLst_t *pTdl; + qDeqID deqIDDel; DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + pTdl = tdlPeek(pThis); /* get current head element */ + if(pTdl == NULL) { /* to-delete list empty */ + DoDeleteBatchFromQStore(pThis, pBatch->nElem); + } else if(pBatch->deqID == pThis->deqIDDel) { + deqIDDel = pThis->deqIDDel; + pTdl = tdlPeek(pThis); + while(pTdl != NULL && deqIDDel == pTdl->deqID) { + DoDeleteBatchFromQStore(pThis, pTdl->nElemDeq); + tdlPop(pThis); + ++deqIDDel; + pTdl = tdlPeek(pThis); + } + /* old entries deleted, now delete current ones... */ + DoDeleteBatchFromQStore(pThis, pBatch->nElem); + } else { + /* can not delete, insert into to-delete list */ + dbgprintf("not at head of to-delete list, enqueue %d\n", (int) pBatch->deqID); + CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem)); + } + +finalize_it: + RETiRet; +} + + +/* Delete a batch of processed user objects from the queue, which includes + * destructing the objects themself. Any entries not marked as finally + * processed are enqueued again. The new enqueue is necessary because we have a + * rgerhards, 2009-05-13 + */ +static inline rsRetVal +DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) +{ + int i; void *pUsr; + int nEnqueued = 0; + rsRetVal localRet; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + for(i = 0 ; i < pBatch->nElem ; ++i) { + pUsr = pBatch->pElem[i].pUsrp; + if( pBatch->pElem[i].state == BATCH_STATE_RDY + || pBatch->pElem[i].state == BATCH_STATE_SUB) { + localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*) pUsr)); + ++nEnqueued; + if(localRet != RS_RET_OK) { + DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet); + } + } + objDestruct(pUsr); + } + + dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued); + + if(nEnqueued > 0) + qqueueChkPersist(pThis, nEnqueued); + + iRet = DeleteBatchFromQStore(pThis, pBatch); + + pBatch->nElem = pBatch->nElemDeq = 0; /* reset batch */ + + RETiRet; +} + + +/* dequeue as many user pointers as are available, until we hit the configured + * upper limit of pointers. Note that this function also deletes all processed + * objects from the previous batch. However, it is perfectly valid that the + * previous batch contained NO objects at all. For example, this happens + * immediately after system startup or when a queue was exhausted and the queue + * worker needed to wait for new data. + * This must only be called when the queue mutex is LOOKED, otherwise serious + * malfunction will happen. + */ +static inline rsRetVal +DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSize) +{ + int nDequeued; + int nDiscarded; + int nDeleted; int iQueueSize; - int bRunsDA; /* cache for early mutex release */ - - /* dequeue element (still protected from mutex) */ - iRet = qqueueDel(pThis, &pUsr); - qqueueChkPersist(pThis); - iQueueSize = qqueueGetOverallQueueSize(pThis); /* cache this for after mutex release */ - bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ - - /* We now need to save the user pointer for the cancel cleanup handler, BUT ONLY - * if we could successfully obtain a user pointer. Otherwise, we would bring the - * cancel cleanup handler into big troubles (and we did ;)). Note that we can - * NOT set the variable further below, as this may lead to an object leak. We - * may get cancelled before we reach that part of the code, so the only - * solution is to do it here. -- rgerhards, 2008-02-27 - */ - if(iRet == RS_RET_OK) { - pWti->pUsrp = pUsr; + void *pUsr; + rsRetVal localRet; + DEFiRet; + + nDeleted = pWti->batch.nElemDeq; + DeleteProcessedBatch(pThis, &pWti->batch); + + nDequeued = nDiscarded = 0; + while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { + CHKiRet(qqueueDeq(pThis, &pUsr)); + + /* check if we should discard this element */ + localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr); + if(localRet == RS_RET_QUEUE_FULL) { + ++nDiscarded; + continue; + } else if(localRet != RS_RET_OK) { + ABORT_FINALIZE(localRet); + } + + /* all well, use this element */ + pWti->batch.pElem[nDequeued].pUsrp = pUsr; + pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY; + ++nDequeued; } + /* it is sufficient to persist only when the bulk of work is done */ + qqueueChkPersist(pThis, nDequeued+nDiscarded+nDeleted); + + pWti->batch.nElem = nDequeued; + pWti->batch.nElemDeq = nDequeued + nDiscarded; + pWti->batch.deqID = getNextDeqID(pThis); + *piRemainingQueueSize = iQueueSize; + +finalize_it: + RETiRet; +} + + +/* dequeue the queued object for the queue consumers. + * rgerhards, 2008-10-21 + * I made a radical change - we now dequeue multiple elements, and store these objects in + * an array of user pointers. We expect that this increases performance. + * rgerhards, 2009-04-22 + */ +static rsRetVal +DequeueConsumable(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + int iQueueSize = 0; /* keep the compiler happy... */ + + /* dequeue element batch (still protected from mutex) */ + iRet = DequeueConsumableElements(pThis, pWti, &iQueueSize); + /* awake some flow-controlled sources if we can do this right now */ /* TODO: this could be done better from a performance point of view -- do it only if * we have someone waiting for the condition (or only when we hit the watermark right * on the nail [exact value]) -- rgerhards, 2008-03-14 + * now that we dequeue batches of pointers, this is much less an issue... + * rgerhards, 2009-04-22 */ - if(iQueueSize < pThis->iFullDlyMrk) { + if(iQueueSize < pThis->iFullDlyMrk / 2) { pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk); } - if(iQueueSize < pThis->iLightDlyMrk) { + if(iQueueSize < pThis->iLightDlyMrk / 2) { pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk); } - /* rgerhards, 2008-09-30: I reversed the order of cond_signal und mutex_unlock - * as of the pthreads recommendation on predictable scheduling behaviour. I don't see - * any problems caused by this, but I add this comment in case some will be seen - * in the next time. - */ + // TODO: MULTI: check physical queue size? pthread_cond_signal(&pThis->notFull); - d_pthread_mutex_unlock(pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); /* WE ARE NO LONGER PROTECTED BY THE MUTEX */ - /* do actual processing (the lengthy part, runs in parallel) - * If we had a problem while dequeing, we do not call the consumer, - * but we otherwise ignore it. This is in the hopes that it will be - * self-healing. However, this is really not a good thing. - * rgerhards, 2008-01-03 - */ - if(iRet != RS_RET_OK) - FINALIZE; - - /* we are running in normal, non-disk-assisted mode do a quick check if we need to drain the queue. - * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to - * provide real-time creation of spool files. - * Note: It is OK to use the cached iQueueSize here, because it does not hurt if it is slightly wrong. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); - -finalize_it: if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) { - dbgoprint((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things " + DBGOPRINT((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things " "may happen\n", iRet); } + RETiRet; } @@ -1529,7 +1515,7 @@ finalize_it: * but you get the idea from the code above. */ static rsRetVal -qqueueRateLimiter(qqueue_t *pThis) +RateLimiter(qqueue_t *pThis) { DEFiRet; int iDelay; @@ -1542,7 +1528,7 @@ qqueueRateLimiter(qqueue_t *pThis) iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ /* time calls are expensive, so only do them when needed */ - time(&tCurr); + datetime.GetTime(&tCurr); localtime_r(&tCurr, &m); iHrCurr = m.tm_hour; @@ -1578,7 +1564,7 @@ qqueueRateLimiter(qqueue_t *pThis) } if(iDelay > 0) { - dbgoprint((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); + DBGOPRINT((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); srSleep(iDelay, 0); } @@ -1586,37 +1572,95 @@ qqueueRateLimiter(qqueue_t *pThis) } +/* This dequeues the next batch. Note that this function must not be + * cancelled, else it will leave back an inconsistent state. + * rgerhards, 2009-05-20 + */ +static inline rsRetVal +DequeueForConsumer(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); + + CHKiRet(DequeueConsumable(pThis, pWti)); + + if(pWti->batch.nElem == 0) + ABORT_FINALIZE(RS_RET_IDLE); + + +finalize_it: + RETiRet; +} + + +/* This is called when a batch is processed and the worker does not + * ask for another batch (e.g. because it is to be terminated) + * rgerhards, 2009-05-27 + */ +static rsRetVal +batchProcessed(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); + + DeleteProcessedBatch(pThis, &pWti->batch); + qqueueChkPersist(pThis, pWti->batch.nElemDeq); + + RETiRet; +} + /* This is the queue consumer in the regular (non-DA) case. It is * protected by the queue mutex, but MUST release it as soon as possible. * rgerhards, 2008-01-21 */ static rsRetVal -qqueueConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerReg(qqueue_t *pThis, wti_t *pWti) { + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->pUsrp)); + CHKiRet(DequeueForConsumer(pThis, pWti)); + + /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ + d_pthread_mutex_unlock(pThis->mut); + + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + + CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch, &pThis->bShutdownImmediate)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 */ +//TODO: MULTIQUEUE: the following setting is no longer correct - need to think about how to do that... if(pThis->iDeqSlowdown) { - dbgoprint((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n", + DBGOPRINT((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n", pThis->iDeqSlowdown); srSleep(pThis->iDeqSlowdown / 1000000, pThis->iDeqSlowdown % 1000000); } + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + + /* now we are done, but need to re-aquire the mutex */ + d_pthread_mutex_lock(pThis->mut); + finalize_it: + dbgprintf("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } -/* This is a special consumer to feed the disk-queue in disk-assited mode. +/* This is a special consumer to feed the disk-queue in disk-assisted mode. * When active, our own queue more or less acts as a memory buffer to the disk. * So this consumer just needs to drain the memory queue and submit entries * to the disk queue. The disk queue will then call the actual consumer from @@ -1626,151 +1670,93 @@ finalize_it: * rgerhards, 2008-01-14 */ static rsRetVal -qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerDA(qqueue_t *pThis, wti_t *pWti) { + int i; + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); + CHKiRet(DequeueForConsumer(pThis, pWti)); + + /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ + d_pthread_mutex_unlock(pThis->mut); + + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + + /* iterate over returned results and enqueue them in DA queue */ + for(i = 0 ; i < pWti->batch.nElem && !pThis->bShutdownImmediate ; i++) { + /* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs + * the message. So far, we simply assume we always have msg_t, what currently is always the case. + * rgerhards, 2009-05-28 + */ +dbgprintf("DA consumer pushes msg '%s'\n", ((msg_t*)(pWti->batch.pElem[i].pUsrp))->pszRawMsg); + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp)))); + pWti->batch.pElem[i].state = BATCH_STATE_COMM; /* commited to other queue! */ + } + + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + + /* now we are done, but need to re-aquire the mutex */ + d_pthread_mutex_lock(pThis->mut); finalize_it: - dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); + DBGOPRINT((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); RETiRet; } /* must only be called when the queue mutex is locked, else results * are not stable! - * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the DA worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue */ -static int +static rsRetVal qqueueChkStopWrkrDA(qqueue_t *pThis) { - /* if our queue is in destruction, we drain to the DA queue and so we shall not terminate - * until we have done so. - */ - int bStopWrkr; - - BEGINfunc + DEFiRet; if(pThis->bEnqOnly) { - bStopWrkr = 1; - } else { - if(pThis->bRunsDA) { - ASSERT(pThis->pqDA != NULL); - if( pThis->pqDA->bEnqOnly - && pThis->pqDA->sizeOnDiskMax > 0 - && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { - /* this queue can never grow, so we can give up... */ - bStopWrkr = 1; - } else if(qqueueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { - bStopWrkr = 1; - } else { - bStopWrkr = 0; - } - } else { - bStopWrkr = 1; - } + iRet = RS_RET_TERMINATE_WHEN_IDLE; } - ENDfunc - return bStopWrkr; + RETiRet; } /* must only be called when the queue mutex is locked, else results * are not stable! * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the regular worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue - */ -static int -qqueueChkStopWrkrReg(qqueue_t *pThis) -{ - return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0); -} - - -/* must only be called when the queue mutex is locked, else results - * are not stable! DA queue version - */ -static int -qqueueIsIdleDA(qqueue_t *pThis) -{ - /* remember: iQueueSize is the DA queue size, not the main queue! */ - /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); -} -/* must only be called when the queue mutex is locked, else results - * are not stable! Regular queue version - */ -static int -qqueueIsIdleReg(qqueue_t *pThis) -{ -#if 0 /* enable for performance testing */ - int ret; - ret = qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); - if(ret) fprintf(stderr, "queue is idle\n"); - return ret; -#else - /* regular code! */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); -#endif -} - - -/* This function is called when a worker thread for the regular queue is shut down. - * If we are the primary queue, this is not really interesting to us. If, however, - * we are the DA (child) queue, that means the DA queue is empty. In that case, we - * need to signal the parent queue's DA worker, so that it can terminate DA mode. - * rgerhards, 2008-01-26 - * rgerhards, 2008-02-27: HOWEVER, in a shutdown condition, it may be that the parent's worker thread pool - * has already been terminated and destructed. This *is* a legal condition and happens - * from time to time in practice. So we need to signal only if there still is a - * parent DA worker queue. Please keep in mind that the the parent's DA worker - * pool is DIFFERENT from our (DA queue) regular worker pool. So when the parent's - * pWtpDA is destructed, there can still be some of our (DAq/wtp) threads be running. - * I am telling this, because I, too, always get confused by those... + * we can terminate. Version for the regular worker thread. */ static rsRetVal -qqueueRegOnWrkrShutdown(qqueue_t *pThis) +ChkStopWrkrReg(qqueue_t *pThis) { DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pqParent != NULL) { - pThis->pqParent->bChildIsDone = 1; /* indicate we are done */ - if(pThis->pqParent->pWtpDA != NULL) { /* see comment in function header from 2008-02-27 */ - wtpAdviseMaxWorkers(pThis->pqParent->pWtpDA, 1); /* reactivate DA worker (always 1) */ - } + if(pThis->bEnqOnly) { + iRet = RS_RET_TERMINATE_NOW; + } else if(pThis->pqParent != NULL) { + iRet = RS_RET_TERMINATE_WHEN_IDLE; } RETiRet; } -/* The following function is called when a regular queue worker starts up. We need this - * hook to indicate in the parent queue (if we are a child) that we are not done yet. +/* return the configured "deq max at once" interval + * rgerhards, 2009-04-22 */ static rsRetVal -qqueueRegOnWrkrStartup(qqueue_t *pThis) +GetDeqBatchSize(qqueue_t *pThis, int *pVal) { DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pqParent != NULL) { - pThis->pqParent->bChildIsDone = 0; - } - + assert(pVal != NULL); + *pVal = pThis->iDeqBatchSize; +if(pThis->pqParent != NULL) // TODO: check why we actually do this! + *pVal = 16; RETiRet; } @@ -1778,11 +1764,10 @@ qqueueRegOnWrkrStartup(qqueue_t *pThis) /* start up the queue - it must have been constructed and parameters defined * before. */ -rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ +rsRetVal +qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; - rsRetVal iRetLocal; - int bInitialized = 0; /* is queue already initialized? */ uchar pszBuf[64]; size_t lenBuf; @@ -1796,11 +1781,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ * influenced by properties which might have been set after queueConstruct () */ if(pThis->pqParent == NULL) { - pThis->mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + pThis->mut = (pthread_mutex_t *) MALLOC (sizeof (pthread_mutex_t)); pthread_mutex_init(pThis->mut, NULL); } else { /* child queue, we need to use parent's mutex */ - dbgoprint((obj_t*) pThis, "I am a child\n"); + DBGOPRINT((obj_t*) pThis, "I am a child\n"); pThis->mut = pThis->pqParent->mut; } @@ -1814,28 +1799,26 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* call type-specific constructor */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ - dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d, " - "full delay %d, light delay %d starting\n", + DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, " + "full delay %d, light delay %d, deq batch size %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, - qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, - pThis->iFullDlyMrk, pThis->iLightDlyMrk); + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), + pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, + pThis->iDeqBatchSize); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ - /* create worker thread pools for regular operation. The DA pool is created on an as-needed - * basis, which potentially means never under most circumstances. + /* create worker thread pools for regular and DA operation. */ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpReg)); CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); - CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRateLimiter)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrReg)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleReg)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerReg)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))qqueueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrStartup)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrShutdown)); + CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerReg)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads)); @@ -1843,27 +1826,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpUsr (pThis->pWtpReg, pThis)); CHKiRet(wtpConstructFinalize (pThis->pWtpReg)); - /* initialize worker thread instances */ - if(pThis->bIsDA) { - /* If we are disk-assisted, we need to check if there is a QIF file - * which we need to load. -- rgerhards, 2008-01-15 - */ - iRetLocal = qqueueHaveQIF(pThis); - if(iRetLocal == RS_RET_OK) { - dbgoprint((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n"); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ - bInitialized = 1; /* we are done */ - } else { - /* TODO: use logerror? -- rgerhards, 2008-01-16 */ - dbgoprint((obj_t*) pThis, "error %d trying to access on-disk queue files, starting without them. " - "Some data may be lost\n", iRetLocal); - } - } + /* set up DA system if we have a disk-assisted queue */ + if(pThis->bIsDA) + InitDA(pThis, LOCK_MUTEX); /* initiate DA mode */ - if(!bInitialized) { - dbgoprint((obj_t*) pThis, "queue starts up without (loading) any DA disk state (this is normal for the DA " - "queue itself!)\n"); - } + DBGOPRINT((obj_t*) pThis, "queue finished initialization\n"); /* if the queue already contains data, we need to start the correct number of worker threads. This can be * the case when a disk queue has been loaded. If we did not start it here, it would never start. @@ -1889,12 +1856,11 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) strm_t *psQIF = NULL; /* Queue Info File */ uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; - obj_t *pUsr; ASSERT(pThis != NULL); if(pThis->qType != QUEUETYPE_DISK) { - if(qqueueGetOverallQueueSize(pThis) > 0) { + if(getPhysicalQueueSize(pThis) > 0) { /* This error code is OK, but we will probably not implement this any time * The reason is that persistence happens via DA queues. But I would like to * leave the code as is, as we so have a hook in case we need one. @@ -1905,28 +1871,29 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) FINALIZE; /* if the queue is empty, we are happy and done... */ } - dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", qqueueGetOverallQueueSize(pThis)); + DBGOPRINT((obj_t*) pThis, "persisting queue to disk, %d entries...\n", getPhysicalQueueSize(pThis)); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - if((bIsCheckpoint != QUEUE_CHECKPOINT) && (qqueueGetOverallQueueSize(pThis) == 0)) { + if((bIsCheckpoint != QUEUE_CHECKPOINT) && (getPhysicalQueueSize(pThis) == 0)) { if(pThis->bNeedDelQIF) { unlink((char*)pszQIFNam); pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + if(pThis->tVars.disk.pReadDel != NULL) /* may be NULL if we had a startup failure! */ + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); FINALIZE; /* nothing left to do, so be happy */ } - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_WRITE)); - CHKiRet(strmSetiAddtlOpenFlags(psQIF, O_TRUNC)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC)); + CHKiRet(strm.SetbSync(psQIF, pThis->bSyncQueueFiles)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, write the property bag for ourselfs * And, surprisingly enough, we currently need to persist only the size of the @@ -1936,29 +1903,19 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) */ CHKiRet(obj.BeginSerializePropBag(psQIF, (obj_t*) pThis)); objSerializeSCALAR(psQIF, iQueueSize, INT); - objSerializeSCALAR(psQIF, iUngottenObjs, INT); objSerializeSCALAR(psQIF, tVars.disk.sizeOnDisk, INT64); objSerializeSCALAR(psQIF, tVars.disk.bytesRead, INT64); CHKiRet(obj.EndSerialize(psQIF)); - /* now we must persist all objects on the ungotten queue - they can not go to - * to the regular files. -- rgerhards, 2008-01-29 - */ - while(pThis->iUngottenObjs > 0) { - CHKiRet(qqueueGetUngottenObj(pThis, &pUsr)); - CHKiRet((objSerialize(pUsr))(pUsr, psQIF)); - objDestruct(pUsr); - } - /* now persist the stream info */ - CHKiRet(strmSerialize(pThis->tVars.disk.pWrite, psQIF)); - CHKiRet(strmSerialize(pThis->tVars.disk.pRead, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pReadDel, psQIF)); /* tell the input file object that it must not delete the file on close if the queue * is non-empty - but only if we are not during a simple checkpoint */ if(bIsCheckpoint != QUEUE_CHECKPOINT) { - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 0)); } /* we have persisted the queue object. So whenever it comes to an empty queue, @@ -1968,28 +1925,75 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); RETiRet; } /* check if we need to persist the current queue info. If an - * error occurs, thus should be ignored by caller (but we still + * error occurs, this should be ignored by caller (but we still * abide to our regular call interface)... * rgerhards, 2008-01-13 + * nUpdates is the number of updates since the last call to this function. + * It may be > 1 due to batches. -- rgerhards, 2009-05-12 */ -rsRetVal qqueueChkPersist(qqueue_t *pThis) +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates) { DEFiRet; - ISOBJ_TYPE_assert(pThis, qqueue); + assert(nUpdates >= 0); + + if(nUpdates == 0) + FINALIZE; - if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { + pThis->iUpdsSincePersist += nUpdates; + if(pThis->iPersistUpdCnt && pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { qqueuePersist(pThis, QUEUE_CHECKPOINT); pThis->iUpdsSincePersist = 0; } +finalize_it: + RETiRet; +} + + +/* persist a queue with all data elements to disk - this is used to handle + * bSaveOnShutdown. We utilize the DA worker to do this. This must only + * be called after all workers have been shut down and if bSaveOnShutdown + * is actually set. Note that this function may potentially run long, + * depending on the queue configuration (e.g. store on remote machine). + * rgerhards, 2009-05-26 + */ +static inline rsRetVal +DoSaveOnShutdown(qqueue_t *pThis) +{ + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + /* we reduce the low water mark, otherwise the DA worker would terminate when + * it is reached. + */ + DBGOPRINT((obj_t*) pThis, "bSaveOnShutdown set, restarting DA worker...\n"); + pThis->bShutdownImmediate = 0; /* would termiante the DA worker! */ + pThis->iLowWtrMrk = 0; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN); /* shutdown worker (only) when done (was _IMMEDIATE!) */ + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* restart DA worker */ + + DBGOPRINT((obj_t*) pThis, "waiting for DA worker to terminate...\n"); + timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); + /* and run the primary queue's DA worker to drain the queue */ + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); + DBGOPRINT((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n", + iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " + "continuing, but results are unpredictable\n", iRetLocal); + } + RETiRet; } @@ -1997,16 +2001,17 @@ rsRetVal qqueueChkPersist(qqueue_t *pThis) /* destructor for the queue object */ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) - pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ - - /* shut down all workers (handles *all* of the persistence logic) - * See function head comment of queueShutdownWorkers () on why we don't call it - * We also do not need to shutdown workers when we are in enqueue-only mode or we are a + /* shut down all workers + * We do not need to shutdown workers when we are in enqueue-only mode or we are a * direct queue - because in both cases we have none... ;) * with a child! -- rgerhards, 2008-01-28 */ if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) - qqueueShutdownWorkers(pThis); + ShutdownWorkers(pThis); + + if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + CHKiRet(DoSaveOnShutdown(pThis)); + } /* finally destruct our (regular) worker thread pool * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, @@ -2042,7 +2047,7 @@ CODESTARTobjDestruct(qqueue) * if need arises (what I doubt...) -- rgerhards, 2008-01-25 */ CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { - dbgoprint((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); + DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); } /* finally, clean up some simple things... */ @@ -2061,11 +2066,8 @@ CODESTARTobjDestruct(qqueue) /* type-specific destructor */ iRet = pThis->qDestruct(pThis); - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); - - if(pThis->pszSpoolDir != NULL) - free(pThis->pszSpoolDir); + free(pThis->pszFilePrefix); + free(pThis->pszSpoolDir); ENDobjDestruct(qqueue) @@ -2079,13 +2081,13 @@ qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) { DEFiRet; - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); + free(pThis->pszFilePrefix); + pThis->pszFilePrefix = NULL; if(pszPrefix == NULL) /* just unset the prefix! */ ABORT_FINALIZE(RS_RET_OK); - if((pThis->pszFilePrefix = malloc(sizeof(uchar) * iLenPrefix + 1)) == NULL) + if((pThis->pszFilePrefix = MALLOC(sizeof(uchar) * iLenPrefix + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFilePrefix, pszPrefix, iLenPrefix + 1); pThis->lenFilePrefix = iLenPrefix; @@ -2115,42 +2117,20 @@ finalize_it: } -/* enqueue a new user data element - * Enqueues the new element and awakes worker thread. +/* enqueue a single data object. + * Note that the queue mutex MUST already be locked when this function is called. + * rgerhards, 2009-06-16 */ -rsRetVal -qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) +static inline rsRetVal +doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) { DEFiRet; - int iCancelStateSave; struct timespec t; - ISOBJ_TYPE_assert(pThis, qqueue); - /* first check if we need to discard this message (which will cause CHKiRet() to exit) - * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize - * and bRunsDA parameters may not reflect the correct settings here, but they are - * "good enough" in the sense that they can be used to drive the decision. Valgrind's - * threading tools may point this access to be an error, but this is done - * intentional. I do not see this causes problems to us. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); - - /* Please note that this function is not cancel-safe and consequently - * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE - * during its execution. If that is not done, race conditions occur if the - * thread is canceled (most important use case is input module termination). - * rgerhards, 2008-01-08 */ - if(pThis->qType != QUEUETYPE_DIRECT) { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(pThis->mut); - } + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); - /* then check if we need to add an assistance disk queue */ - if(pThis->bIsDA) - CHKiRet(qqueueChkStrtDA(pThis)); - /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. * Basic flow control has always been implemented and protects the queue structures @@ -2173,12 +2153,12 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) */ if(flowCtlType == eFLOWCTL_FULL_DELAY) { while(pThis->iQueueSize >= pThis->iFullDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */ } } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) { if(pThis->iQueueSize >= pThis->iLightDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */ } @@ -2192,10 +2172,11 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize) || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0 && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { - dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); timeoutComp(&t, pThis->toEnq); +// TODO : handle enqOnly => discard! if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { - dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); objDestruct(pUsr); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } @@ -2203,7 +2184,39 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis); + +finalize_it: + RETiRet; +} + +/* enqueue multiple user data elements at once. The aim is to provide a faster interface + * for object submission. Uses the multi_submit_t helper object. + * Please note that this function is not cancel-safe and consequently + * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE + * during its execution. If that is not done, race conditions occur if the + * thread is canceled (most important use case is input module termination). + * rgerhards, 2009-06-16 + */ +rsRetVal +qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub) +{ + int iCancelStateSave; + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pMultiSub != NULL); + + if(pThis->qType != QUEUETYPE_DIRECT) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); + } + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { + CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i])); + } + + qqueueChkPersist(pThis, pMultiSub->nElem); finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { @@ -2212,73 +2225,49 @@ finalize_it: /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); - dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); - /* the following pthread_yield is experimental, but brought us performance - * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 - * rgerhards, 2008-10-09 - * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 - */ - if(pThis->bOptimizeUniProc) - pthread_yield(); + DBGOPRINT((obj_t*) pThis, "MultiEnqObj advised worker start\n"); } RETiRet; } -/* set queue mode to enqueue only or not - * There is one subtle issue: this method may be called during queue - * construction or while it is running. In the former case, the queue - * mutex does not yet exist (it is NULL), while in the later case it - * must be locked. The function detects the state and operates as - * required. - * rgerhards, 2008-01-16 +/* enqueue a new user data element + * Enqueues the new element and awakes worker thread. */ -static rsRetVal -qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +rsRetVal +qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) { DEFiRet; - DEFVARS_mutexProtection; + int iCancelStateSave; ISOBJ_TYPE_assert(pThis, qqueue); - /* for simplicity, we do one big mutex lock. This method is extremely seldom - * called, so that doesn't matter... -- rgerhards, 2008-01-16 - */ - if(pThis->mut != NULL) { - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); + if(pThis->qType != QUEUETYPE_DIRECT) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); } - if(bEnqOnly == pThis->bEnqOnly) - FINALIZE; /* no change, nothing to do */ - - if(pThis->bQueueStarted) { - /* we need to adjust queue operation only if we are not during initial param setup */ - if(bEnqOnly == 1) { - /* switch to enqueue-only mode */ - /* this means we need to terminate all workers - that's it... */ - dbgoprint((obj_t*) pThis, "switching to enqueue-only mode, terminating all worker threads\n"); - if(pThis->pWtpReg != NULL) - wtpWakeupAllWrkr(pThis->pWtpReg); - if(pThis->pWtpDA != NULL) - wtpWakeupAllWrkr(pThis->pWtpDA); - } else { - /* switch back to regular mode */ - ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); /* we don't need this so far... */ - } - } + CHKiRet(doEnqSingleObj(pThis, flowCtlType, pUsr)); - pThis->bEnqOnly = bEnqOnly; + qqueueChkPersist(pThis, 1); finalize_it: - if(pThis->mut != NULL) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + if(pThis->qType != QUEUETYPE_DIRECT) { + /* make sure at least one worker is running. */ + qqueueAdviseMaxWorkers(pThis); + /* and release the mutex */ + d_pthread_mutex_unlock(pThis->mut); + pthread_setcancelstate(iCancelStateSave, NULL); + DBGOPRINT((obj_t*) pThis, "EnqueueMsg advised worker start\n"); } + RETiRet; } /* some simple object access methods */ +DEFpropSetMeth(qqueue, bSyncQueueFiles, int) DEFpropSetMeth(qqueue, iPersistUpdCnt, int) DEFpropSetMeth(qqueue, iDeqtWinFromHr, int) DEFpropSetMeth(qqueue, iDeqtWinToHr, int) @@ -2296,6 +2285,7 @@ DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int) DEFpropSetMeth(qqueue, bSaveOnShutdown, int) DEFpropSetMeth(qqueue, pUsr, void*) DEFpropSetMeth(qqueue, iDeqSlowdown, int) +DEFpropSetMeth(qqueue, iDeqBatchSize, int) DEFpropSetMeth(qqueue, sizeOnDiskMax, int64) @@ -2314,8 +2304,6 @@ static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp) if(isProp("iQueueSize")) { pThis->iQueueSize = pProp->val.num; - } else if(isProp("iUngottenObjs")) { - pThis->iUngottenObjs = pProp->val.num; } else if(isProp("tVars.disk.sizeOnDisk")) { pThis->tVars.disk.sizeOnDisk = pProp->val.num; } else if(isProp("tVars.disk.bytesRead")) { @@ -2340,6 +2328,9 @@ rsRetVal qqueueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/queue.h b/runtime/queue.h index a267862d..38c0d491 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -27,8 +27,18 @@ #include <pthread.h> #include "obj.h" #include "wtp.h" +#include "batch.h" #include "stream.h" +/* support for the toDelete list */ +typedef struct toDeleteLst_s toDeleteLst_t; +struct toDeleteLst_s { + qDeqID deqID; + int nElemDeq; /* numbe of elements that were dequeued and as such must now be discarded */ + struct toDeleteLst_s *pNext; +}; + + /* queue types */ typedef enum { QUEUETYPE_FIXED_ARRAY = 0,/* a simple queue made out of a fixed (initially malloced) array fast but memoryhog */ @@ -44,25 +54,15 @@ typedef struct qLinkedList_S { } qLinkedList_t; -typedef struct qWrkThrd_s { - pthread_t thrdID; /* thread ID */ - qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - obj_t *pUsr; /* current user object being processed (or NULL if none) */ - struct queue_s *pQueue; /* my queue (important if only the work thread instance is passed! */ - int iThrd; /* my worker thread array index */ - pthread_cond_t condInitDone; /* signaled when the thread startup is done (once per thread existance) */ - pthread_mutex_t mut; -} qWrkThrd_t; /* type for queue worker threads */ - /* the queue object */ -typedef struct queue_s { +struct queue_s { BEGINobjInstance; queueType_t qType; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ - int bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ - int bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ - int bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ - int bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ + int nLogDeq; /* number of elements currently logically dequeued */ + int bShutdownImmediate; /* should all workers cease processing messages? */ + sbool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ + sbool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ + sbool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ int iQueueSize; /* Current number of elements in the queue */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ @@ -73,17 +73,20 @@ typedef struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ + sbool bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ int iFullDlyMrk; /* if the queue is above this mark, FULL_DELAYable message are put on hold */ int iLightDlyMrk; /* if the queue is above this mark, LIGHT_DELAYable message are put on hold */ int iDiscardSeverity;/* messages of this severity above are discarded on too-full queue */ - int bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ + sbool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ int toQShutdown; /* timeout for regular queue shutdown in ms */ int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ + toDeleteLst_t *toDeleteLst;/* this queue's to-delete list */ int toEnq; /* enqueue timeout */ + int iDeqBatchSize; /* max number of elements that shall be dequeued at once */ /* rate limiting settings (will be expanded) */ int iDeqSlowdown; /* slow down dequeue by specified nbr of microseconds */ /* end rate limiting */ @@ -97,18 +100,19 @@ typedef struct queue_s { * applied to detect user configuration errors (and tell me how should we detect what * the user really wanted...). -- rgerhards, 2008-04-02 */ - /* ane dequeue time window */ - rsRetVal (*pConsumer)(void *,void*); /* user-supplied consumer function for dequeued messages */ + /* end dequeue time window */ + rsRetVal (*pConsumer)(void *,batch_t*,int*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the - * user pointer that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 is pointer - * to message) - * rgerhards, 2008-01-28 + * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 + * is pointer to an array of message message pointers), arg3 is a pointer to an interger which is zero + * during normal operations and one if the consumer must urgently shut down. */ /* type-specific handlers (set during construction) */ rsRetVal (*qConstruct)(struct queue_s *pThis); rsRetVal (*qDestruct)(struct queue_s *pThis); rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr); - rsRetVal (*qDel)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDel)(struct queue_s *pThis); /* end type-specific handler */ /* synchronization variables */ pthread_mutex_t mutThrdMgmt; /* mutex for the queue's thread management */ @@ -117,7 +121,6 @@ typedef struct queue_s { pthread_cond_t belowFullDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t belowLightDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t condDAReady;/* signalled when the DA queue is fully initialized and ready for processing */ - int bChildIsDone; /* set to 1 when the child DA queue has finished processing, 0 otherwise */ int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* the following variables are always present, because they @@ -132,42 +135,33 @@ typedef struct queue_s { int iNumberFiles; /* how many files make up the queue? */ int64 iMaxFileSize; /* max size for a single queue file */ int64 sizeOnDiskMax; /* maximum size on disk allowed */ + qDeqID deqIDAdd; /* next dequeue ID to use during add to queue store */ + qDeqID deqIDDel; /* queue store delete position */ int bIsDA; /* is this queue disk assisted? */ - int bRunsDA; /* is this queue actually *running* disk assisted? */ struct queue_s *pqDA; /* queue for disk-assisted modes */ struct queue_s *pqParent;/* pointer to the parent (if this is a child queue) */ int bDAEnqOnly; /* EnqOnly setting for DA queue */ - /* some data elements for the queueUngetObj() functionality. This list should always be short - * and is always kept in memory - */ - qLinkedList_t *pUngetRoot; - qLinkedList_t *pUngetLast; - int iUngottenObjs; /* number of objects currently in the "ungotten" list */ /* now follow queueing mode specific data elements */ union { /* different data elements based on queue type (qType) */ struct { - long head, tail; + long deqhead, head, tail; void** pBuf; /* the queued user data structure */ } farray; struct { - qLinkedList_t *pRoot; + qLinkedList_t *pDeqRoot; + qLinkedList_t *pDelRoot; qLinkedList_t *pLast; } linklist; struct { int64 sizeOnDisk; /* current amount of disk space used */ int64 bytesRead; /* number of bytes read from current (undeleted!) file */ - strm_t *pWrite; /* current file to be written */ - strm_t *pRead; /* current file to be read */ + strm_t *pWrite; /* current file to be written */ + strm_t *pReadDeq; /* current file for dequeueing */ + strm_t *pReadDel; /* current file for deleting */ } disk; } tVars; -} qqueue_t; - -/* some symbolic constants for easier reference */ -#define QUEUE_MODE_ENQDEQ 0 -#define QUEUE_MODE_ENQONLY 1 +}; -#define QUEUE_IDX_DA_WORKER 0 /* index for the DA worker (fixed) */ -#define QUEUE_PTR_DA_WORKER(x) (&((pThis)->pWrkThrds[0])) /* the define below is an "eternal" timeout for the timeout settings which require a value. * It is one day, which is not really eternal, but comes close to it if we think about @@ -178,14 +172,16 @@ typedef struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); +rsRetVal qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub); rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*)); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); +PROTOTYPEpropSetMeth(qqueue, bSyncQueueFiles, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinToHr, int); PROTOTYPEpropSetMeth(qqueue, toQShutdown, long); @@ -201,6 +197,7 @@ PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int); PROTOTYPEpropSetMeth(qqueue, pUsr, void*); PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int); PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); +PROTOTYPEpropSetMeth(qqueue, iDeqBatchSize, int); #define qqueueGetID(pThis) ((unsigned long) pThis) #endif /* #ifndef QUEUE_H_INCLUDED */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 8df100a1..a76ae25a 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -77,6 +77,10 @@ #include "conf.h" #include "glbl.h" #include "errmsg.h" +#include "prop.h" +#include "rule.h" +#include "ruleset.h" +#include "parser.h" /* forward definitions */ static rsRetVal dfltErrLogger(int, uchar *errMsg); @@ -144,20 +148,16 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) * class immediately after it is initialized. And, of course, we load those classes * first that we use ourselfs... -- rgerhards, 2008-03-07 */ + if(ppErrObj != NULL) *ppErrObj = "prop"; + CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; CHKiRet(glblClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "datetime"; - CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "str,"; - CHKiRet(strmClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wti"; - CHKiRet(wtiClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wtp"; - CHKiRet(wtpClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "queue"; - CHKiRet(qqueueClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok_token"; + CHKiRet(ctok_tokenClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok"; + CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmstk"; CHKiRet(vmstkClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "sysvar"; @@ -168,14 +168,22 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(vmopClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmprg"; CHKiRet(vmprgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok_token"; - CHKiRet(ctok_tokenClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok"; - CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "expr"; CHKiRet(exprClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "rule"; + CHKiRet(ruleClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ruleset"; + CHKiRet(rulesetClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wti"; + CHKiRet(wtiClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wtp"; + CHKiRet(wtpClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "queue"; + CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "parser"; + CHKiRet(parserClassInit(NULL)); /* dummy "classes" */ if(ppErrObj != NULL) *ppErrObj = "str"; @@ -206,6 +214,8 @@ rsrtExit(void) /* do actual de-init only if we are the last runtime user */ confClassExit(); glblClassExit(); + rulesetClassExit(); + ruleClassExit(); objClassExit(); /* *THIS* *MUST/SHOULD?* always be the first class initilizer being called (except debug)! */ } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 77d845fd..a75d2bc0 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -29,7 +29,29 @@ /* ############################################################# * * # Config Settings # * * ############################################################# */ -#define RS_STRINGBUF_ALLOC_INCREMENT 128 +#define RS_STRINGBUF_ALLOC_INCREMENT 128 +/* MAXSIZE are absolute maxima, while BUFSIZE are just values after which + * processing is more time-intense. The BUFSIZE params currently add their + * value to the fixed size of the message object. + */ +#define CONF_TAG_MAXSIZE 512 /* a value that is deemed far too large for any valid TAG */ +#define CONF_HOSTNAME_MAXSIZE 512 /* a value that is deemed far too large for any valid HOSTNAME */ +#define CONF_RAWMSG_BUFSIZE 101 +#define CONF_TAG_BUFSIZE 32 +#define CONF_HOSTNAME_BUFSIZE 32 +#define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ +#define CONF_MIN_SIZE_FOR_COMPRESS 60 /* config param: minimum message size to try compression. The smaller + * the message, the less likely is any compression gain. We check for + * gain before we submit the message. But to do so we still need to + * do the (costly) compress() call. The following setting sets a size + * for which no call to compress() is done at all. This may result in + * a few more bytes being transmited but better overall performance. + * Note: I have not yet checked the minimum UDP packet size. It might be + * that we do not save anything by compressing very small messages, because + * UDP might need to pad ;) + * rgerhards, 2006-11-30 + */ + /* ############################################################# * * # End Config Settings # * @@ -58,15 +80,34 @@ #endif +/* the rsyslog core provides information about present feature to plugins + * asking it. Below are feature-test macros which must be used to query + * features. Note that this must be powers of two, so that multiple queries + * can be combined. -- rgerhards, 2009-04-27 + */ +#define CORE_FEATURE_BATCHING 1 +/*#define CORE_FEATURE_whatever 2 ... and so on ... */ + +/* some universal fixed size integer defines ... */ +typedef long long int64; +typedef long long unsigned uint64; +typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef char intTiny; /* 0..127! */ +typedef unsigned char uintTiny; /* 0..255! */ + /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ +typedef struct aUsrp_s aUsrp_t; typedef struct thrdInfo thrdInfo_t; typedef struct obj_s obj_t; -typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ +typedef struct ruleset_s ruleset_t; +typedef struct rule_s rule_t; +//typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ typedef struct NetAddr netAddr_t; typedef struct netstrms_s netstrms_t; typedef struct netstrm_s netstrm_t; typedef struct nssel_s nssel_t; +typedef struct nspoll_s nspoll_t; typedef enum nsdsel_waitOp_e nsdsel_waitOp_t; typedef struct nsd_ptcp_s nsd_ptcp_t; typedef struct nsd_gtls_s nsd_gtls_t; @@ -74,9 +115,14 @@ typedef struct nsd_gsspi_s nsd_gsspi_t; typedef struct nsd_nss_s nsd_nss_t; typedef struct nsdsel_ptcp_s nsdsel_ptcp_t; typedef struct nsdsel_gtls_s nsdsel_gtls_t; +typedef struct nsdpoll_ptcp_s nsdpoll_ptcp_t; +typedef struct wti_s wti_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; +typedef obj_t nsdpoll_t; typedef struct msg msg_t; +typedef struct queue_s qqueue_t; +typedef struct prop_s prop_t; typedef struct interface_s interface_t; typedef struct objInfo_s objInfo_t; typedef enum rsRetVal_ rsRetVal; /**< friendly type for global return value */ @@ -85,21 +131,30 @@ typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; typedef struct tcps_sess_s tcps_sess_t; +typedef struct strmsrv_s strmsrv_t; +typedef struct strms_sess_s strms_sess_t; typedef struct vmstk_s vmstk_t; +typedef struct batch_obj_s batch_obj_t; +typedef struct batch_s batch_t; +typedef struct wtp_s wtp_t; +typedef struct modInfo_s modInfo_t; +typedef struct parser_s parser_t; +typedef struct parserList_s parserList_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ +typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? - -/* some universal 64 bit define... */ -typedef long long int64; -typedef long long unsigned uint64; -typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif +typedef struct epoll_event epoll_event_t; + +typedef char sbool; /* (small bool) I intentionally use char, to keep it slim so that many fit into the CPU cache! */ + /* settings for flow control * TODO: is there a better place for them? -- rgerhards, 2008-03-14 */ @@ -109,6 +164,72 @@ typedef enum { eFLOWCTL_FULL_DELAY = 2 /**< delay possible for extended period of time */ } flowControl_t; +/* filter operations */ +typedef enum { + FIOP_NOP = 0, /* do not use - No Operation */ + FIOP_CONTAINS = 1, /* contains string? */ + FIOP_ISEQUAL = 2, /* is (exactly) equal? */ + FIOP_STARTSWITH = 3, /* starts with a string? */ + FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ + FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ +} fiop_t; + + +/* multi-submit support. + * This is done via a simple data structure, which holds the number of elements + * as well as an array of to-be-submitted messages. + * rgerhards, 2009-06-16 + */ +typedef struct multi_submit_s multi_submit_t; +struct multi_submit_s { + short maxElem; /* maximum number of Elements */ + short nElem; /* current number of Elements, points to the next one FREE */ + msg_t **ppMsgs; +}; + + +#ifndef _PATH_CONSOLE +#define _PATH_CONSOLE "/dev/console" +#endif + +/* properties are now encoded as (tiny) integers. I do not use an enum as I would like + * to keep the memory footprint small (and thus cache hits high). + * rgerhards, 2009-06-26 + */ +typedef uintTiny propid_t; +#define PROP_INVALID 0 +#define PROP_MSG 1 +#define PROP_TIMESTAMP 2 +#define PROP_HOSTNAME 3 +#define PROP_SYSLOGTAG 4 +#define PROP_RAWMSG 5 +#define PROP_INPUTNAME 6 +#define PROP_FROMHOST 7 +#define PROP_FROMHOST_IP 8 +#define PROP_PRI 9 +#define PROP_PRI_TEXT 10 +#define PROP_IUT 11 +#define PROP_SYSLOGFACILITY 12 +#define PROP_SYSLOGFACILITY_TEXT 13 +#define PROP_SYSLOGSEVERITY 14 +#define PROP_SYSLOGSEVERITY_TEXT 15 +#define PROP_TIMEGENERATED 16 +#define PROP_PROGRAMNAME 17 +#define PROP_PROTOCOL_VERSION 18 +#define PROP_STRUCTURED_DATA 19 +#define PROP_APP_NAME 20 +#define PROP_PROCID 21 +#define PROP_MSGID 22 +#define PROP_SYS_NOW 150 +#define PROP_SYS_YEAR 151 +#define PROP_SYS_MONTH 152 +#define PROP_SYS_DAY 153 +#define PROP_SYS_HOUR 154 +#define PROP_SYS_HHOUR 155 +#define PROP_SYS_QHOUR 156 +#define PROP_SYS_MINUTE 157 +#define PROP_SYS_MYHOSTNAME 158 + /* The error codes below are orginally "borrowed" from * liblogging. As such, we reserve values up to -2999 @@ -270,16 +391,45 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ + RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ + RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ + RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */ + RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */ + RS_RET_NON_SIZELIMITCMD = -2125, /**< size limit for file defined, but no size limit command given */ + RS_RET_SIZELIMITCMD_DIDNT_RESOLVE = -2126, /**< size limit command did not resolve situation */ + RS_RET_STREAM_DISABLED = -2127, /**< a file has been disabled (e.g. by size limit restriction) */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ + RS_RET_ZLIB_ERR = -2141, /**< error during zlib call */ + RS_RET_VAR_NOT_FOUND = -2142, /**< variable not found */ + RS_RET_EMPTY_MSG = -2143, /**< provided (raw) MSG is empty */ + RS_RET_PEER_CLOSED_CONN = -2144, /**< remote peer closed connection (information, no error) */ + RS_RET_NO_SRCNAME_TPL = -2150, /**< sourcename template was not specified where one was needed (omudpspoof spoof addr) */ + RS_RET_HOST_NOT_SPECIFIED = -2151, /**< (target) host was not specified where it was needed */ + RS_RET_ERR_LIBNET_INIT = -2152, /**< error initializing libnet */ + RS_RET_FORCE_TERM = -2153, /**< thread was forced to terminate by bShallShutdown, a state, not an error */ + RS_RET_RULES_QUEUE_EXISTS = -2154,/**< we were instructed to create a new ruleset queue, but one already exists */ + RS_RET_NO_CURR_RULESET = -2155,/**< no current ruleset exists (but one is required) */ + RS_RET_NO_MSG_PASSING = -2156,/**< output module interface parameter passing mode "MSG" is not available but required */ + RS_RET_RULESET_NOT_FOUND = -2157,/**< a required ruleset could not be found */ + RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */ + RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */ + RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */ + RS_RET_EINTR = -2161, /**< EINTR occured during a system call (not necessarily an error) */ + RS_RET_ERR_EPOLL = -2162, /**< epoll() returned with an unexpected error code */ + RS_RET_ERR_EPOLL_CTL = -2163, /**< epol_ctll() returned with an unexpected error code */ + RS_RET_TIMEOUT = -2164, /**< timeout occured during operation */ + RS_RET_RCV_ERR = -2165, /**< error occured during socket rcv operation */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ /* some generic error/status codes */ + RS_RET_OK = 0, /**< operation successful */ RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */ RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */ RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */ - RS_RET_OK = 0 /**< operation successful */ + RS_RET_IDLE = 4, /**< operation successful, but callee is idle (e.g. because queue is empty) */ + RS_RET_TERMINATE_WHEN_IDLE = 5 /**< operation successful, function is requested to terminate when idle */ }; /* some helpful macros to work with srRetVals. @@ -359,6 +509,10 @@ typedef enum rsObjectID rsObjID; # define O_CLOEXEC 0 #endif +/* some constants */ +#define MUTEX_ALREADY_LOCKED 0 +#define LOCK_MUTEX 1 + /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); diff --git a/runtime/rule.c b/runtime/rule.c new file mode 100644 index 00000000..65ad071e --- /dev/null +++ b/runtime/rule.c @@ -0,0 +1,479 @@ +/* rule.c - rsyslog's rule object + * + * See file comment in rule.c for the overall structure of rule processing. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <ctype.h> + +#include "rsyslog.h" +#include "obj.h" +#include "action.h" +#include "rule.h" +#include "errmsg.h" +#include "vm.h" +#include "var.h" +#include "srUtils.h" +#include "unicode-helper.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(expr) +DEFobjCurrIf(var) +DEFobjCurrIf(vm) + + +/* support for simple textual representation of FIOP names + * rgerhards, 2005-09-27 + */ +static char* +getFIOPName(unsigned iFIOP) +{ + char *pRet; + switch(iFIOP) { + case FIOP_CONTAINS: + pRet = "contains"; + break; + case FIOP_ISEQUAL: + pRet = "isequal"; + break; + case FIOP_STARTSWITH: + pRet = "startswith"; + break; + case FIOP_REGEX: + pRet = "regex"; + break; + default: + pRet = "NOP"; + break; + } + return pRet; +} + + +/* iterate over all actions, this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + return llExecFunc(&pThis->llActList, pFunc, pParam); +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +typedef struct processMsgDoActions_s { + int bPrevWasSuspended; /* was the previous action suspended? */ + msg_t *pMsg; +} processMsgDoActions_t; +DEFFUNC_llExecFunc(processMsgDoActions) +{ + DEFiRet; + rsRetVal iRetMod; /* return value of module - we do not always pass that back */ + action_t *pAction = (action_t*) pData; + processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; + + assert(pAction != NULL); + + if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { + dbgprintf("not calling action because the previous one is not suspended\n"); + ABORT_FINALIZE(RS_RET_OK); + } + + iRetMod = actionCallAction(pAction, pDoActData->pMsg); + if(iRetMod == RS_RET_DISCARDMSG) { + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } else if(iRetMod == RS_RET_SUSPENDED) { + /* indicate suspension for next module to be called */ + pDoActData->bPrevWasSuspended = 1; + } else { + pDoActData->bPrevWasSuspended = 0; + } + +finalize_it: + RETiRet; +} + + +/* This functions looks at the given message and checks if it matches the + * provided filter condition. + */ +static rsRetVal +shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) +{ + DEFiRet; + unsigned short pbMustBeFreed; + uchar *pszPropVal; + int bRet = 0; + size_t propLen; + vm_t *pVM = NULL; + var_t *pResult = NULL; + + ISOBJ_TYPE_assert(pRule, rule); + assert(pMsg != NULL); + + /* we first have a look at the global, BSD-style block filters (for tag + * and host). Only if they match, we evaluate the actual filter. + * rgerhards, 2005-10-18 + */ + if(pRule->eHostnameCmpMode == HN_NO_COMP) { + /* EMPTY BY INTENSION - we check this value first, because + * it is the one most often used, so this saves us time! + */ + } else if(pRule->eHostnameCmpMode == HN_COMP_MATCH) { + if(rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '+%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } else { /* must be -hostname */ + if(!rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '-%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } + +RUNLOG_VAR("%p", pRule->pCSProgNameComp); + if(pRule->pCSProgNameComp != NULL) { + int bInv = 0, bEqv = 0, offset = 0; + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp)) == '-') { + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp) + 1) == '-') + offset = 1; + else { + bInv = 1; + offset = 1; + } + } + if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, + (uchar*) getProgramName(pMsg, LOCK_MUTEX), getProgramNameLen(pMsg, LOCK_MUTEX))) + bEqv = 1; + + if((!bEqv && !bInv) || (bEqv && bInv)) { + /* not equal or inverted selection, so we are already done... */ + DBGPRINTF("programname filter '%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg, LOCK_MUTEX)); + FINALIZE; + } + } + + /* done with the BSD-style block filters */ + + if(pRule->f_filter_type == FILTER_PRI) { + /* skip messages that are incorrect priority */ +dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility]); + if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ + ((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) ) + bRet = 0; + else + bRet = 1; + } else if(pRule->f_filter_type == FILTER_EXPR) { + CHKiRet(vm.Construct(&pVM)); + CHKiRet(vm.ConstructFinalize(pVM)); + CHKiRet(vm.SetMsg(pVM, pMsg)); + CHKiRet(vm.ExecProg(pVM, pRule->f_filterData.f_expr->pVmprg)); + CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); + dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + /* VM is destructed on function exit */ + bRet = (pResult->val.num) ? 1 : 0; + } else { + assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ + pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.propID, &propLen, &pbMustBeFreed); + + /* Now do the compares (short list currently ;)) */ + switch(pRule->f_filterData.prop.operation ) { + case FIOP_CONTAINS: + if(rsCStrLocateInSzStr(pRule->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) + bRet = 1; + break; + case FIOP_ISEQUAL: + if(rsCStrSzStrCmp(pRule->f_filterData.prop.pCSCompValue, + pszPropVal, ustrlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_STARTSWITH: + if(rsCStrSzStrStartsWithCStr(pRule->f_filterData.prop.pCSCompValue, + pszPropVal, ustrlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_REGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 0, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 1, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + default: + /* here, it handles NOP (for performance reasons) */ + assert(pRule->f_filterData.prop.operation == FIOP_NOP); + bRet = 1; /* as good as any other default ;) */ + break; + } + + /* now check if the value must be negated */ + if(pRule->f_filterData.prop.isNegated) + bRet = (bRet == 1) ? 0 : 1; + + if(Debug) { + dbgprintf("Filter: check for property '%s' (value '%s') ", + propIDToName(pRule->f_filterData.prop.propID), pszPropVal); + if(pRule->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("%s '%s': %s\n", + getFIOPName(pRule->f_filterData.prop.operation), + rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue), + bRet ? "TRUE" : "FALSE"); + } + + /* cleanup */ + if(pbMustBeFreed) + free(pszPropVal); + } + +finalize_it: + /* destruct in any case, not just on error, but it makes error handling much easier */ + if(pVM != NULL) + vm.Destruct(&pVM); + + if(pResult != NULL) + var.Destruct(&pResult); + + *bProcessMsg = bRet; + RETiRet; +} + + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(rule_t *pThis, msg_t *pMsg) +{ + int bProcessMsg; + processMsgDoActions_t DoActData; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, rule); + assert(pMsg != NULL); + + /* first check the filters... */ + CHKiRet(shouldProcessThisMessage(pThis, pMsg, &bProcessMsg)); + if(bProcessMsg) { + DoActData.pMsg = pMsg; + DoActData.bPrevWasSuspended = 0; + CHKiRet(llExecFunc(&pThis->llActList, processMsgDoActions, (void*)&DoActData)); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(rule) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(rule) + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +ruleConstructFinalize(rule_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + + /* note: actionDestruct is from action.c API! */ + CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +/* destructor for the rule object */ +BEGINobjDestruct(rule) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(rule) + if(pThis->pCSHostnameComp != NULL) + rsCStrDestruct(&pThis->pCSHostnameComp); + if(pThis->pCSProgNameComp != NULL) + rsCStrDestruct(&pThis->pCSProgNameComp); + + if(pThis->f_filter_type == FILTER_PROP) { + if(pThis->f_filterData.prop.pCSCompValue != NULL) + rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); + if(pThis->f_filterData.prop.regex_cache != NULL) + rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); + } else if(pThis->f_filter_type == FILTER_EXPR) { + if(pThis->f_filterData.f_expr != NULL) + expr.Destruct(&pThis->f_filterData.f_expr); + } + + llDestroy(&pThis->llActList); +ENDobjDestruct(rule) + + +/* set the associated ruleset */ +static rsRetVal +setAssRuleset(rule_t *pThis, ruleset_t *pRuleset) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + ISOBJ_TYPE_assert(pRuleset, ruleset); + pThis->pRuleset = pRuleset; + RETiRet; +} + +/* get the associated ruleset (may be NULL if not set!) */ +static ruleset_t* +getAssRuleset(rule_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, rule); + return pThis->pRuleset; +} + + +/* helper to DebugPrint, to print out all actions via + * the llExecFunc() facility. + */ +DEFFUNC_llExecFunc(dbgPrintInitInfoAction) +{ + DEFiRet; + iRet = actionDbgPrint((action_t*) pData); + dbgprintf("\n"); + RETiRet; +} + + +/* debugprint for the rule object */ +BEGINobjDebugPrint(rule) /* be sure to specify the object type also in END and CODESTART macros! */ + int i; +CODESTARTobjDebugPrint(rule) + dbgoprint((obj_t*) pThis, "rsyslog rule:\n"); + if(pThis->pCSProgNameComp != NULL) + dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(pThis->pCSProgNameComp)); + if(pThis->eHostnameCmpMode != HN_NO_COMP) + dbgprintf("hostname: %s '%s'\n", + pThis->eHostnameCmpMode == HN_COMP_MATCH ? + "only" : "allbut", + rsCStrGetSzStrNoNULL(pThis->pCSHostnameComp)); + if(pThis->f_filter_type == FILTER_PRI) { + for (i = 0; i <= LOG_NFACILITIES; i++) + if (pThis->f_filterData.f_pmask[i] == TABLE_NOPRI) + dbgprintf(" X "); + else + dbgprintf("%2X ", pThis->f_filterData.f_pmask[i]); + } else if(pThis->f_filter_type == FILTER_EXPR) { + dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); + } else { + dbgprintf("PROPERTY-BASED Filter:\n"); + dbgprintf("\tProperty.: '%s'\n", propIDToName(pThis->f_filterData.prop.propID)); + dbgprintf("\tOperation: "); + if(pThis->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("'%s'\n", getFIOPName(pThis->f_filterData.prop.operation)); + dbgprintf("\tValue....: '%s'\n", + rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue)); + dbgprintf("\tAction...: "); + } + + dbgprintf("\nActions:\n"); + llExecFunc(&pThis->llActList, dbgPrintInitInfoAction, NULL); /* actions */ + + dbgprintf("\n"); +ENDobjDebugPrint(rule) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(rule) +CODESTARTobjQueryInterface(rule) + if(pIf->ifVersion != ruleCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = ruleConstruct; + pIf->ConstructFinalize = ruleConstructFinalize; + pIf->Destruct = ruleDestruct; + pIf->DebugPrint = ruleDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->ProcessMsg = processMsg; + pIf->SetAssRuleset = setAssRuleset; + pIf->GetAssRuleset = getAssRuleset; +finalize_it: +ENDobjQueryInterface(rule) + + +/* Exit the rule class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(rule, OBJ_IS_CORE_MODULE) /* class, version */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(expr, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vm, CORE_COMPONENT); +ENDObjClassExit(rule) + + +/* Initialize the rule class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(rule, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(expr, CORE_COMPONENT)); + CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, ruleDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, ruleConstructFinalize); +ENDObjClassInit(rule) + +/* vi:set ai: + */ diff --git a/runtime/rule.h b/runtime/rule.h new file mode 100644 index 00000000..7b607637 --- /dev/null +++ b/runtime/rule.h @@ -0,0 +1,77 @@ +/* The rule object. + * + * This implements rules within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULE_H +#define INCLUDED_RULE_H + +#include "linkedlist.h" +#include "regexp.h" +#include "expr.h" + +/* the rule object */ +struct rule_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + /* filter properties */ + enum { + FILTER_PRI = 0, /* traditional PRI based filer */ + FILTER_PROP = 1, /* extended filter, property based */ + FILTER_EXPR = 2 /* extended filter, expression based */ + } f_filter_type; + EHostnameCmpMode eHostnameCmpMode; + cstr_t *pCSHostnameComp; /* hostname to check */ + cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ + union { + u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ + struct { + fiop_t operation; + regex_t *regex_cache; /* cache for compiled REs, if such are used */ + cstr_t *pCSCompValue; /* value to "compare" against */ + sbool isNegated; + propid_t propID; /* ID of the requested property */ + } prop; + expr_t *f_expr; /* expression object */ + } f_filterData; + + ruleset_t *pRuleset; /* associated ruleset */ + linkedList_t llActList; /* list of configured actions */ +}; + +/* interfaces */ +BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(rule); + rsRetVal (*Construct)(rule_t **ppThis); + rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(rule_t **ppThis); + rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); + rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg); + rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*); + ruleset_t* (*GetAssRuleset)(rule_t *pThis); +ENDinterface(rule) +#define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(rule); + +#endif /* #ifndef INCLUDED_RULE_H */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c new file mode 100644 index 00000000..1a77be2b --- /dev/null +++ b/runtime/ruleset.c @@ -0,0 +1,564 @@ +/* ruleset.c - rsyslog's ruleset object + * + * We have a two-way structure of linked lists: one global linked list + * (llAllRulesets) hold alls rule sets that we know. Included in each + * list is a list of rules (which contain a list of actions, but that's + * a different story). + * + * Usually, only a single rule set is executed. However, there exist some + * situations where all rules must be iterated over, for example on HUP. Thus, + * we also provide interfaces to do that. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <ctype.h> + +#include "rsyslog.h" +#include "obj.h" +#include "cfsysline.h" +#include "msg.h" +#include "ruleset.h" +#include "rule.h" +#include "errmsg.h" +#include "parser.h" +#include "unicode-helper.h" +#include "dirty.h" /* for main ruleset queue creation */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(rule) +DEFobjCurrIf(parser) + +linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ +ruleset_t *pCurrRuleset = NULL; /* currently "active" ruleset */ +ruleset_t *pDfltRuleset = NULL; /* current default ruleset, e.g. for binding to actions which have no other */ + +/* ---------- linked-list key handling functions ---------- */ + +/* destructor for linked list keys. + */ +static rsRetVal keyDestruct(void __attribute__((unused)) *pData) +{ + free(pData); + return RS_RET_OK; +} + + +/* ---------- END linked-list key handling functions ---------- */ + + +/* driver to iterate over all of this ruleset actions */ +typedef struct iterateAllActions_s { + rsRetVal (*pFunc)(void*, void*); + void *pParam; +} iterateAllActions_t; +DEFFUNC_llExecFunc(doIterateRulesetActions) +{ + DEFiRet; + rule_t* pRule = (rule_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over all actions of THIS rule set. + */ +static rsRetVal +iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&(pThis->llRules), doIterateRulesetActions, ¶ms)); + +finalize_it: + RETiRet; +} + + +/* driver to iterate over all actions */ +DEFFUNC_llExecFunc(doIterateAllActions) +{ + DEFiRet; + ruleset_t* pThis = (ruleset_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over ALL actions present in the WHOLE system. + * this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + +finalize_it: + RETiRet; +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +DEFFUNC_llExecFunc(processMsgDoRules) +{ + rsRetVal iRet; + ISOBJ_TYPE_assert(pData, rule); + iRet = rule.ProcessMsg((rule_t*) pData, (msg_t*) pParam); +dbgprintf("ruleset: get iRet %d from rule.ProcessMsg()\n", iRet); + return iRet; +} + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(msg_t *pMsg) +{ + ruleset_t *pThis; + DEFiRet; + assert(pMsg != NULL); + + pThis = (pMsg->pRuleset == NULL) ? pDfltRuleset : pMsg->pRuleset; + ISOBJ_TYPE_assert(pThis, ruleset); + + CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); + +finalize_it: +dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet); + RETiRet; +} + + +/* return the ruleset-assigned parser list. NULL means use the default + * parser list. + * rgerhards, 2009-11-04 + */ +static parserList_t* +GetParserList(msg_t *pMsg) +{ + return (pMsg->pRuleset == NULL) ? pDfltRuleset->pParserLst : pMsg->pRuleset->pParserLst; +} + + +/* Add a new rule to the end of the current rule set. We do a number + * of checks and ignore the rule if it does not pass them. + */ +static rsRetVal +addRule(ruleset_t *pThis, rule_t **ppRule) +{ + int iActionCnt; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, ruleset); + ISOBJ_TYPE_assert(*ppRule, rule); + + CHKiRet(llGetNumElts(&(*ppRule)->llActList, &iActionCnt)); + if(iActionCnt == 0) { + errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); + rule.Destruct(ppRule); + } else { + CHKiRet(llAppend(&pThis->llRules, NULL, *ppRule)); + dbgprintf("selector line successfully processed\n"); + } + +finalize_it: + RETiRet; +} + + +/* set name for ruleset */ +static rsRetVal setName(ruleset_t *pThis, uchar *pszName) +{ + DEFiRet; + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); + +finalize_it: + RETiRet; +} + + +/* get current ruleset + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static ruleset_t* +GetCurrent(void) +{ + return pCurrRuleset; +} + + +/* get main queue associated with ruleset. If no ruleset-specifc main queue + * is set, the primary main message queue is returned. + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static qqueue_t* +GetRulesetQueue(ruleset_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, ruleset); + return (pThis->pQueue == NULL) ? pMsgQueue : pThis->pQueue; +} + + +/* Find the ruleset with the given name and return a pointer to its object. + */ +static rsRetVal +GetRuleset(ruleset_t **ppRuleset, uchar *pszName) +{ + DEFiRet; + assert(ppRuleset != NULL); + assert(pszName != NULL); + + CHKiRet(llFind(&llRulesets, pszName, (void*) ppRuleset)); + +finalize_it: + RETiRet; +} + + +/* Set a new default rule set. If the default can not be found, no change happens. + */ +static rsRetVal +SetDefaultRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pDfltRuleset = pRuleset; + dbgprintf("default rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + +/* Set a new current rule set. If the ruleset can not be found, no change happens. + */ +static rsRetVal +SetCurrRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pCurrRuleset = pRuleset; + dbgprintf("current rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + +/* destructor we need to destruct rules inside our linked list contents. + */ +static rsRetVal +doRuleDestruct(void *pData) +{ + rule_t *pRule = (rule_t *) pData; + DEFiRet; + rule.Destruct(&pRule); + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(ruleset) /* be sure to specify the object type also in END macro! */ + CHKiRet(llInit(&pThis->llRules, doRuleDestruct, NULL, NULL)); +finalize_it: +ENDobjConstruct(ruleset) + + +/* ConstructionFinalizer + * This also adds the rule set to the list of all known rulesets. + */ +static rsRetVal +rulesetConstructFinalize(ruleset_t *pThis) +{ + uchar *keyName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + + /* we must duplicate our name, as the key destructer would also + * free it, resulting in a double-free. It's also cleaner to have + * two separate copies. + */ + CHKmalloc(keyName = ustrdup(pThis->pszName)); + CHKiRet(llAppend(&llRulesets, keyName, pThis)); + + /* this now also is the new current ruleset */ + pCurrRuleset = pThis; + + /* and also the default, if so far none has been set */ + if(pDfltRuleset == NULL) + pDfltRuleset = pThis; + +finalize_it: + RETiRet; +} + + +/* destructor for the ruleset object */ +BEGINobjDestruct(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(ruleset) + dbgprintf("destructing ruleset %p, name %p\n", pThis, pThis->pszName); + if(pThis->pQueue != NULL) { + qqueueDestruct(&pThis->pQueue); + } + if(pThis->pParserLst != NULL) { + parser.DestructParserList(&pThis->pParserLst); + } + llDestroy(&pThis->llRules); + free(pThis->pszName); +ENDobjDestruct(ruleset) + +/* this is a special destructor for the linkedList class. LinkedList does NOT + * provide a pointer to the pointer, but rather the raw pointer itself. So we + * must map this, otherwise the destructor will abort. + */ +static rsRetVal +rulesetDestructForLinkedList(void *pData) +{ + ruleset_t *pThis = (ruleset_t*) pData; + return rulesetDestruct(&pThis); +} + + +/* destruct ALL rule sets that reside in the system. This must + * be callable before unloading this module as the module may + * not be unloaded before unload of the actions is required. This is + * kind of a left-over from previous logic and may be optimized one + * everything runs stable again. -- rgerhards, 2009-06-10 + */ +static rsRetVal +destructAllActions(void) +{ + DEFiRet; + + CHKiRet(llDestroy(&llRulesets)); + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + +finalize_it: + RETiRet; +} + +/* helper for debugPrint(), initiates rule printing */ +DEFFUNC_llExecFunc(doDebugPrintRule) +{ + return rule.DebugPrint((rule_t*) pData); +} +/* debugprint for the ruleset object */ +BEGINobjDebugPrint(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(ruleset) + dbgoprint((obj_t*) pThis, "rsyslog ruleset %s:\n", pThis->pszName); + llExecFunc(&pThis->llRules, doDebugPrintRule, NULL); +ENDobjDebugPrint(ruleset) + + +/* helper for debugPrintAll(), prints a single ruleset */ +DEFFUNC_llExecFunc(doDebugPrintAll) +{ + return rulesetDebugPrint((ruleset_t*) pData); +} +/* debug print all rulesets + */ +static rsRetVal +debugPrintAll(void) +{ + DEFiRet; + dbgprintf("All Rulesets:\n"); + llExecFunc(&llRulesets, doDebugPrintAll, NULL); + dbgprintf("End of Rulesets.\n"); + RETiRet; +} + + +/* Create a ruleset-specific "main" queue for this ruleset. If one is already + * defined, an error message is emitted but nothing else is done. + * Note: we use the main message queue parameters for queue creation and access + * syslogd.c directly to obtain these. This is far from being perfect, but + * considered acceptable for the time being. + * rgerhards, 2009-10-27 + */ +static rsRetVal +rulesetCreateQueue(void __attribute__((unused)) *pVal, int *pNewVal) +{ + DEFiRet; + + if(pCurrRuleset == NULL) { + errmsg.LogError(0, RS_RET_NO_CURR_RULESET, "error: currently no specific ruleset specified, thus a " + "queue can not be added to it"); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } + + if(pCurrRuleset->pQueue != NULL) { + errmsg.LogError(0, RS_RET_RULES_QUEUE_EXISTS, "error: ruleset already has a main queue, can not " + "add another one"); + ABORT_FINALIZE(RS_RET_RULES_QUEUE_EXISTS); + } + + if(pNewVal == 0) + FINALIZE; /* if it is turned off, we do not need to change anything ;) */ + + dbgprintf("adding a ruleset-specific \"main\" queue"); + CHKiRet(createMainQueue(&pCurrRuleset->pQueue, UCHAR_CONSTANT("ruleset"))); + +finalize_it: + RETiRet; +} + + +/* Add a ruleset specific parser to the ruleset. Note that adding the first + * parser automatically disables the default parsers. If they are needed as well, + * the must be added via explicit config directives. + * Note: this is the only spot in the code that requires the parser object. In order + * to solve some class init bootstrap sequence problems, we get the object handle here + * instead of during module initialization. Note that objUse() is capable of being + * called multiple times. + * rgerhards, 2009-11-04 + */ +static rsRetVal +rulesetAddParser(void __attribute__((unused)) *pVal, uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + assert(pCurrRuleset != NULL); + + CHKiRet(objUse(parser, CORE_COMPONENT)); + iRet = parser.FindParser(&pParser, pName); + if(iRet == RS_RET_PARSER_NOT_FOUND) { + errmsg.LogError(0, RS_RET_PARSER_NOT_FOUND, "error: parser '%s' unknown at this time " + "(maybe defined too late in rsyslog.conf?)", pName); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } else if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "error trying to find parser '%s'\n", pName); + FINALIZE; + } + + CHKiRet(parser.AddParserToList(&pCurrRuleset->pParserLst, pParser)); + + dbgprintf("added parser '%s' to ruleset '%s'\n", pName, pCurrRuleset->pszName); +RUNLOG_VAR("%p", pCurrRuleset->pParserLst); + +finalize_it: + d_free(pName); /* no longer needed */ + + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(ruleset) +CODESTARTobjQueryInterface(ruleset) + if(pIf->ifVersion != rulesetCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = rulesetConstruct; + pIf->ConstructFinalize = rulesetConstructFinalize; + pIf->Destruct = rulesetDestruct; + pIf->DebugPrint = rulesetDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->DestructAllActions = destructAllActions; + pIf->AddRule = addRule; + pIf->ProcessMsg = processMsg; + pIf->SetName = setName; + pIf->DebugPrintAll = debugPrintAll; + pIf->GetCurrent = GetCurrent; + pIf->GetRuleset = GetRuleset; + pIf->SetDefaultRuleset = SetDefaultRuleset; + pIf->SetCurrRuleset = SetCurrRuleset; + pIf->GetRulesetQueue = GetRulesetQueue; + pIf->GetParserList = GetParserList; +finalize_it: +ENDobjQueryInterface(ruleset) + + +/* Exit the ruleset class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ + llDestroy(&llRulesets); + objRelease(errmsg, CORE_COMPONENT); + objRelease(rule, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); +ENDObjClassExit(ruleset) + + +/* Initialize the ruleset class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(rule, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, rulesetDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, rulesetConstructFinalize); + + /* prepare global data */ + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + + /* config file handlers */ + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetparser", 0, eCmdHdlrGetWord, rulesetAddParser, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetcreatemainqueue", 0, eCmdHdlrBinary, rulesetCreateQueue, NULL, NULL)); +ENDObjClassInit(ruleset) + +/* vi:set ai: + */ diff --git a/runtime/ruleset.h b/runtime/ruleset.h new file mode 100644 index 00000000..222d773e --- /dev/null +++ b/runtime/ruleset.h @@ -0,0 +1,66 @@ +/* The ruleset object. + * + * This implements rulesets within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULESET_H +#define INCLUDED_RULESET_H + +#include "queue.h" +#include "linkedlist.h" + +/* the ruleset object */ +struct ruleset_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ + uchar *pszName; /* name of our ruleset */ + qqueue_t *pQueue; /* "main" message queue, if the ruleset has its own (else NULL) */ + parserList_t *pParserLst;/* list of parsers to use for this ruleset */ +}; + +/* interfaces */ +BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(ruleset); + rsRetVal (*DebugPrintAll)(void); + rsRetVal (*Construct)(ruleset_t **ppThis); + rsRetVal (*ConstructFinalize)(ruleset_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(ruleset_t **ppThis); + rsRetVal (*IterateAllActions)(rsRetVal (*pFunc)(void*, void*), void* pParam); + rsRetVal (*DestructAllActions)(void); + rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); + rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); + rsRetVal (*ProcessMsg)(msg_t *pMsg); + rsRetVal (*GetRuleset)(ruleset_t **ppThis, uchar*); + rsRetVal (*SetDefaultRuleset)(uchar*); + rsRetVal (*SetCurrRuleset)(uchar*); + ruleset_t* (*GetCurrent)(void); + qqueue_t* (*GetRulesetQueue)(ruleset_t*); + /* v3, 2009-11-04 */ + parserList_t* (*GetParserList)(msg_t *); +ENDinterface(ruleset) +#define rulesetCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(ruleset); + +#endif /* #ifndef INCLUDED_RULESET_H */ diff --git a/runtime/srUtils.h b/runtime/srUtils.h index bfce4cbb..c4f73e16 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -92,6 +92,7 @@ void srSleep(int iSeconds, int iuSeconds); char *rs_strerror_r(int errnum, char *buf, size_t buflen); int decodeSyslogName(uchar *name, syslogName_t *codetab); int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); +rsRetVal getFileSize(uchar *pszName, off_t *pSize); /* mutex operations */ /* some macros to cancel-safe lock a mutex (it will automatically be released @@ -108,20 +109,18 @@ int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); #define mutex_cancelsafe_unlock(mut) pthread_cleanup_pop(1) /* some useful constants */ -#define MUTEX_ALREADY_LOCKED 0 -#define LOCK_MUTEX 1 #define DEFVARS_mutexProtection\ - int iCancelStateSave; \ int bLockedOpIsLocked=0 #define BEGIN_MTX_PROTECTED_OPERATIONS(mut, bMustLock) \ if(bMustLock == LOCK_MUTEX) { \ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ d_pthread_mutex_lock(mut); \ + assert(bLockedOpIsLocked == 0); \ bLockedOpIsLocked = 1; \ } #define END_MTX_PROTECTED_OPERATIONS(mut) \ if(bLockedOpIsLocked) { \ d_pthread_mutex_unlock(mut); \ - pthread_setcancelstate(iCancelStateSave, NULL); \ + bLockedOpIsLocked = 0; \ } + #endif diff --git a/runtime/srutils.c b/runtime/srutils.c index d01ca20d..2bed624e 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -46,6 +46,9 @@ #include "srUtils.h" #include "obj.h" +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif /* here we host some syslog specific names. There currently is no better place * to do it, but over here is also not ideal... -- rgerhards, 2008-02-14 @@ -158,7 +161,7 @@ uchar *srUtilStrDup(uchar *pOld, size_t len) assert(pOld != NULL); - if((pNew = malloc(len + 1)) != NULL) + if((pNew = MALLOC(len + 1)) != NULL) memcpy(pNew, pOld, len + 1); return pNew; @@ -183,7 +186,7 @@ int makeFileParentDirs(uchar *szFile, size_t lenFile, mode_t mode, assert(lenFile > 0); len = lenFile + 1; /* add one for '\0'-byte */ - if((pszWork = malloc(sizeof(uchar) * len)) == NULL) + if((pszWork = MALLOC(sizeof(uchar) * len)) == NULL) return -1; memcpy(pszWork, szFile, len); for(p = pszWork+1 ; *p ; p++) @@ -326,7 +329,7 @@ rsRetVal genFileName(uchar **ppName, uchar *pDirName, size_t lenDirName, uchar * } lenName = lenDirName + 1 + lenFName + lenBuf + 1; /* last +1 for \0 char! */ - if((pName = malloc(sizeof(uchar) * lenName)) == NULL) + if((pName = MALLOC(sizeof(uchar) * lenName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); /* got memory, now construct string */ @@ -366,20 +369,34 @@ int getNumberDigits(long lNum) /* compute an absolute time timeout suitable for calls to pthread_cond_timedwait() + * iTimeout is in milliseconds * rgerhards, 2008-01-14 */ rsRetVal timeoutComp(struct timespec *pt, long iTimeout) { +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ + +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, pt); +# else + gettimeofday(&tv, NULL); + pt->tv_sec = tv.tv_sec; + pt->tv_nsec = tv.tv_usec * 1000; +# endif + pt->tv_sec += iTimeout / 1000; pt->tv_nsec += (iTimeout % 1000) * 1000000; /* think INTEGER arithmetic! */ if(pt->tv_nsec > 999999999) { /* overrun? */ pt->tv_nsec -= 1000000000; + ++pt->tv_sec; } - pt->tv_sec += iTimeout / 1000; ENDfunc return RS_RET_OK; /* so far, this is static... */ } @@ -395,11 +412,21 @@ timeoutVal(struct timespec *pt) { struct timespec t; long iTimeout; - BEGINfunc +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif iTimeout = (pt->tv_nsec - t.tv_nsec) / 1000000; iTimeout += (pt->tv_sec - t.tv_sec) * 1000; @@ -553,6 +580,33 @@ int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep) } +/* get the size of a file or return appropriate error code. If an error is returned, + * *pSize content is undefined. + * rgerhards, 2009-06-12 + */ +rsRetVal +getFileSize(uchar *pszName, off_t *pSize) +{ + int ret; + struct stat statBuf; + DEFiRet; + + ret = stat((char*) pszName, &statBuf); + if(ret == -1) { + switch(errno) { + case EACCES: ABORT_FINALIZE(RS_RET_NO_FILE_ACCESS); + case ENOTDIR: + case ENOENT: ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + default: ABORT_FINALIZE(RS_RET_FILE_NO_STAT); + } + } + + *pSize = statBuf.st_size; + +finalize_it: + RETiRet; +} + /* vim:set ai: */ diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..f565dc90 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1,4 +1,3 @@ -//TODO: O_TRUC mode! /* The serial stream class. * * A serial stream provides serial data access. In theory, serial streams @@ -7,8 +6,17 @@ * "driver"). * * File begun on 2008-01-09 by RGerhards + * Large modifications in 2009-06 to support using it with omfile, including zip writer. + * Note that this file obtains the zlib wrapper object is needed, but it never frees it + * again. While this sounds like a leak (and one may argue it actually is), there is no + * harm associated with that. The reason is that strm is a core object, so it is terminated + * only when rsyslogd exists. As we could only release on termination (or else bear more + * overhead for keeping track of how many users we have), not releasing zlibw is OK, because + * it will be released when rsyslogd terminates. We may want to revisit this decision if + * it turns out to be problematic. Then, we need to quasi-refcount the number of accesses + * to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -39,24 +47,198 @@ #include <unistd.h> #include <sys/stat.h> /* required for HP UX */ #include <errno.h> +#include <pthread.h> #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" #include "obj.h" #include "stream.h" +#include "unicode-helper.h" +#include "module-template.h" +#if HAVE_SYS_PRCTL_H +# include <sys/prctl.h> +#endif + +#define inline /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(zlibw) + +/* forward definitions */ +static rsRetVal strmFlush(strm_t *pThis); +static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal strmCloseFile(strm_t *pThis); +static void *asyncWriterThread(void *pPtr); +static rsRetVal doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); + /* methods */ -/* first, we define type-specific handlers. The provide a generic functionality, +/* Try to resolve a size limit situation. This is used to support custom-file size handlers + * for omfile. It first runs the command, and then checks if we are still above the size + * treshold. Note that this works only with single file names, NOT with circular names. + * Note that pszCurrFName can NOT be taken from pThis, because the stream is closed when + * we are called (and that destroys pszCurrFName, as there is NO CURRENT file name!). So + * we need to receive the name as a parameter. + * initially wirtten 2005-06-21, moved to this class & updates 2009-06-01, both rgerhards + */ +static rsRetVal +resolveFileSizeLimit(strm_t *pThis, uchar *pszCurrFName) +{ + uchar *pParams; + uchar *pCmd; + uchar *p; + off_t actualFileSize; + rsRetVal localRet; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + assert(pszCurrFName != NULL); + + if(pThis->pszSizeLimitCmd == NULL) { + ABORT_FINALIZE(RS_RET_NON_SIZELIMITCMD); /* nothing we can do in this case... */ + } + + /* we first check if we have command line parameters. We assume this, + * when we have a space in the program name. If we find it, everything after + * the space is treated as a single argument. + */ + CHKmalloc(pCmd = ustrdup(pThis->pszSizeLimitCmd)); + + for(p = pCmd ; *p && *p != ' ' ; ++p) { + /* JUST SKIP */ + } + + if(*p == ' ') { + *p = '\0'; /* pretend string-end */ + pParams = p+1; + } else + pParams = NULL; + + /* the execProg() below is probably not great, but at least is is + * fairly secure now. Once we change the way file size limits are + * handled, we should also revisit how this command is run (and + * with which parameters). rgerhards, 2007-07-20 + */ + execProg(pCmd, 1, pParams); + + free(pCmd); + + localRet = getFileSize(pszCurrFName, &actualFileSize); + + if(localRet == RS_RET_OK && actualFileSize >= pThis->iSizeLimit) { + ABORT_FINALIZE(RS_RET_SIZELIMITCMD_DIDNT_RESOLVE); /* OK, it didn't work out... */ + } else if(localRet != RS_RET_FILE_NOT_FOUND) { + /* file not found is OK, the command may have moved away the file */ + ABORT_FINALIZE(localRet); + } + +finalize_it: + if(iRet != RS_RET_OK) { + if(iRet == RS_RET_SIZELIMITCMD_DIDNT_RESOLVE) { + DBGPRINTF("file size limit cmd for file '%s' did no resolve situation\n", pszCurrFName); + } else { + DBGPRINTF("file size limit cmd for file '%s' failed with code %d.\n", pszCurrFName, iRet); + } + pThis->bDisabled = 1; + } + + RETiRet; +} + + +/* Check if the file has grown beyond the configured omfile iSizeLimit + * and, if so, initiate processing. + */ +static rsRetVal +doSizeLimitProcessing(strm_t *pThis) +{ + uchar *pszCurrFName = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + ASSERT(pThis->iSizeLimit != 0); + ASSERT(pThis->fd != -1); + + if(pThis->iCurrOffs >= pThis->iSizeLimit) { + /* strmClosefile() destroys the current file name, so we + * need to preserve it. + */ + CHKmalloc(pszCurrFName = ustrdup(pThis->pszCurrFName)); + CHKiRet(strmCloseFile(pThis)); + CHKiRet(resolveFileSizeLimit(pThis, pszCurrFName)); + } + +finalize_it: + free(pszCurrFName); + RETiRet; +} + + +/* now, we define type-specific handlers. The provide a generic functionality, * but for this specific type of strm. The mapping to these handlers happens during * strm construction. Later on, handlers are called by pointers present in the * strm instance object. */ +/* do the physical open() call on a file. + */ +static rsRetVal +doPhysOpen(strm_t *pThis) +{ + int iFlags = 0; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + + /* compute which flags we need to provide to open */ + switch(pThis->tOperationsMode) { + case STREAMMODE_READ: + iFlags = O_CLOEXEC | O_NOCTTY | O_RDONLY; + break; + case STREAMMODE_WRITE: /* legacy mode used inside queue engine */ + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT; + break; + case STREAMMODE_WRITE_TRUNC: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_TRUNC; + break; + case STREAMMODE_WRITE_APPEND: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_APPEND; + break; + default:assert(0); + break; + } + if(pThis->sType == STREAMTYPE_NAMED_PIPE) { + DBGPRINTF("Note: stream '%s' is a named pipe, open with O_NONBLOCK\n", pThis->pszCurrFName); + iFlags |= O_NONBLOCK; + } + + pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); + DBGPRINTF("file '%s' opened as #%d with mode %d\n", pThis->pszCurrFName, pThis->fd, pThis->tOpenMode); + if(pThis->fd == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + dbgoprint((obj_t*) pThis, "open error %d, file '%s': %s\n", errno, pThis->pszCurrFName, errStr); + if(err == ENOENT) + ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + else + ABORT_FINALIZE(RS_RET_IO_ERROR); + } else { + if(!ustrcmp(pThis->pszCurrFName, UCHAR_CONSTANT(_PATH_CONSOLE)) || isatty(pThis->fd)) { + DBGPRINTF("file %d is a tty-type file\n", pThis->fd); + pThis->bIsTTY = 1; + } else { + pThis->bIsTTY = 0; + } + } + +finalize_it: + RETiRet; +} + + /* open a strm file * It is OK to call this function when the stream is already open. In that * case, it returns immediately with RS_RET_OK @@ -64,10 +246,8 @@ DEFobjStaticHelpers static rsRetVal strmOpenFile(strm_t *pThis) { DEFiRet; - int iFlags; ASSERT(pThis != NULL); - ASSERT(pThis->tOperationsMode == STREAMMODE_READ || pThis->tOperationsMode == STREAMMODE_WRITE); if(pThis->fd != -1) ABORT_FINALIZE(RS_RET_OK); @@ -80,7 +260,7 @@ static rsRetVal strmOpenFile(strm_t *pThis) pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); } else { if(pThis->pszDir == NULL) { - if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) + if((pThis->pszCurrFName = ustrdup(pThis->pszFName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } else { CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, @@ -88,34 +268,41 @@ static rsRetVal strmOpenFile(strm_t *pThis) } } - /* compute which flags we need to provide to open */ - if(pThis->tOperationsMode == STREAMMODE_READ) - iFlags = O_RDONLY; - else - iFlags = O_WRONLY | O_CREAT; - - iFlags |= pThis->iAddtlOpenFlags; - - pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); - if(pThis->fd == -1) { - int ierrnoSave = errno; - dbgoprint((obj_t*) pThis, "open error %d, file '%s'\n", errno, pThis->pszCurrFName); - if(ierrnoSave == ENOENT) - ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); - else - ABORT_FINALIZE(RS_RET_IO_ERROR); - } + CHKiRet(doPhysOpen(pThis)); pThis->iCurrOffs = 0; + if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) { + /* we need to obtain the current offset */ + off_t offset; + CHKiRet(getFileSize(pThis->pszCurrFName, &offset)); + pThis->iCurrOffs = offset; + } - dbgoprint((obj_t*) pThis, "opened file '%s' for %s (0x%x) as %d\n", pThis->pszCurrFName, - (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", iFlags, pThis->fd); + dbgoprint((obj_t*) pThis, "opened file '%s' for %s as %d\n", pThis->pszCurrFName, + (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", pThis->fd); finalize_it: RETiRet; } +/* wait for the output writer thread to be done. This must be called before actions + * that require data to be persisted. May be called in non-async mode and is a null + * operation than. Must be called with the mutex locked. + */ +static inline void +strmWaitAsyncWriterDone(strm_t *pThis) +{ + BEGINfunc + if(pThis->bAsyncWrite) { + /* awake writer thread and make it write out everything */ + pthread_cond_signal(&pThis->notEmpty); + d_pthread_cond_wait(&pThis->isEmpty, &pThis->mut); + } + ENDfunc +} + + /* close a strm file * Note that the bDeleteOnClose flag is honored. If it is set, the file will be * deleted after close. This is in support for the qRead thread. @@ -128,14 +315,33 @@ static rsRetVal strmCloseFile(strm_t *pThis) ASSERT(pThis->fd != -1); dbgoprint((obj_t*) pThis, "file %d closing\n", pThis->fd); - if(pThis->tOperationsMode == STREAMMODE_WRITE) - strmFlush(pThis); + if(!pThis->bInClose && pThis->tOperationsMode != STREAMMODE_READ) { + pThis->bInClose = 1; + if(pThis->bAsyncWrite) { + strmFlush(pThis); + } else { + strmWaitAsyncWriterDone(pThis); + } + pThis->bInClose = 0; + } - close(pThis->fd); // TODO: error check + close(pThis->fd); pThis->fd = -1; + if(pThis->fdDir != -1) { + /* close associated directory handle, if it is open */ + close(pThis->fdDir); + pThis->fdDir = -1; + } + if(pThis->bDeleteOnClose) { - unlink((char*) pThis->pszCurrFName); // TODO: check returncode + if(unlink((char*) pThis->pszCurrFName) == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("error %d unlinking '%s' - ignored: %s\n", + errno, pThis->pszCurrFName, errStr); + } } pThis->iCurrOffs = 0; /* we are back at begin of file */ @@ -229,15 +435,12 @@ strmHandleEOF(strm_t *pThis) ISOBJ_TYPE_assert(pThis, strm); switch(pThis->sType) { case STREAMTYPE_FILE_SINGLE: + case STREAMTYPE_NAMED_PIPE: ABORT_FINALIZE(RS_RET_EOF); break; case STREAMTYPE_FILE_CIRCULAR: /* we have multiple files and need to switch to the next one */ /* TODO: think about emulating EOF in this case (not yet needed) */ -#if 0 - if(pThis->iMaxFiles == 0) /* TODO: why do we need this? ;) */ - ABORT_FINALIZE(RS_RET_EOF); -#endif dbgoprint((obj_t*) pThis, "file %d EOF\n", pThis->fd); CHKiRet(strmNextFile(pThis)); break; @@ -295,7 +498,7 @@ finalize_it: * NOTE: needs to be enhanced to support sticking with a strm entry (if not * deleted). */ -rsRetVal strmReadChar(strm_t *pThis, uchar *pC) +static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) { DEFiRet; @@ -329,7 +532,7 @@ finalize_it: * character buffering capability. * rgerhards, 2008-01-07 */ -rsRetVal strmUnreadChar(strm_t *pThis, uchar c) +static rsRetVal strmUnreadChar(strm_t *pThis, uchar c) { ASSERT(pThis != NULL); ASSERT(pThis->iUngetC == -1); @@ -351,7 +554,7 @@ rsRetVal strmUnreadChar(strm_t *pThis, uchar c) * are pthread_killed() upon termination. So if we use their native pointer, they * can cleanup (but only then). */ -rsRetVal +static rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr) { DEFiRet; @@ -360,19 +563,19 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr) ASSERT(pThis != NULL); ASSERT(ppCStr != NULL); - CHKiRet(rsCStrConstruct(ppCStr)); + CHKiRet(cstrConstruct(ppCStr)); /* now read the line */ CHKiRet(strmReadChar(pThis, &c)); while(c != '\n') { - CHKiRet(rsCStrAppendChar(*ppCStr, c)); + CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } - CHKiRet(rsCStrFinish(*ppCStr)); + CHKiRet(cstrFinalize(*ppCStr)); finalize_it: if(iRet != RS_RET_OK && *ppCStr != NULL) - rsCStrDestruct(ppCStr); + cstrDestruct(ppCStr); RETiRet; } @@ -383,26 +586,75 @@ finalize_it: BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! */ pThis->iCurrFNum = 1; pThis->fd = -1; + pThis->fdDir = -1; pThis->iUngetC = -1; pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); - pThis->tOpenMode = 0600; /* TODO: make configurable */ + pThis->tOpenMode = 0600; ENDobjConstruct(strm) /* ConstructionFinalizer * rgerhards, 2008-01-09 */ -rsRetVal strmConstructFinalize(strm_t *pThis) +static rsRetVal strmConstructFinalize(strm_t *pThis) { + rsRetVal localRet; + int i; DEFiRet; ASSERT(pThis != NULL); - if(pThis->pIOBuf == NULL) { /* allocate our io buffer in case we have not yet */ - if((pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - pThis->iBufPtrMax = 0; /* results in immediate read request */ + pThis->iBufPtrMax = 0; /* results in immediate read request */ + if(pThis->iZipLevel) { /* do we need a zip buf? */ + localRet = objUse(zlibw, LM_ZLIBW_FILENAME); + if(localRet != RS_RET_OK) { + pThis->iZipLevel = 0; + DBGPRINTF("stream was requested with zip mode, but zlibw module unavailable (%d) - using " + "without zip\n", localRet); + } else { + /* we use the same size as the original buf, as we would like + * to make sure we can write out everything with a SINGLE api call! + * We add another 128 bytes to take care of the gzip header and "all eventualities". + */ + CHKmalloc(pThis->pZipBuf = (Bytef*) MALLOC(sizeof(uchar) * pThis->sIOBufSize + 128)); + } + } + + /* if we are aset to sync, we must obtain a file handle to the directory for fsync() purposes */ + if(pThis->bSync && !pThis->bIsTTY) { + pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY | O_CLOEXEC | O_NOCTTY); + if(pThis->fdDir == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("error %d opening directory file for fsync() use - fsync for directory disabled: %s\n", + errno, errStr); + } + } + + /* if we have a flush interval, we need to do async writes in any case */ + if(pThis->iFlushInterval != 0) { + pThis->bAsyncWrite = 1; + } + + /* if we work asynchronously, we need a couple of synchronization objects */ + if(pThis->bAsyncWrite) { + pthread_mutex_init(&pThis->mut, 0); + pthread_cond_init(&pThis->notFull, 0); + pthread_cond_init(&pThis->notEmpty, 0); + pthread_cond_init(&pThis->isEmpty, 0); + pThis->iCnt = pThis->iEnq = pThis->iDeq = 0; + for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) { + CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); + } + pThis->pIOBuf = pThis->asyncBuf[0].pBuf; + pThis->bStopWriter = 0; + if(pthread_create(&pThis->writerThreadID, NULL, asyncWriterThread, pThis) != 0) + DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis); + } else { + /* we work synchronously, so we need to alloc a fixed pIOBuf */ + CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); } finalize_it: @@ -410,24 +662,57 @@ finalize_it: } +/* stop the writer thread (we MUST be runnnig asynchronously when this method + * is called!). Note that the mutex must be locked! -- rgerhards, 2009-07-06 + */ +static inline void +stopWriter(strm_t *pThis) +{ + BEGINfunc + pThis->bStopWriter = 1; + pthread_cond_signal(&pThis->notEmpty); + d_pthread_mutex_unlock(&pThis->mut); + pthread_join(pThis->writerThreadID, NULL); + ENDfunc +} + + /* destructor for the strm object */ BEGINobjDestruct(strm) /* be sure to specify the object type also in END and CODESTART macros! */ + int i; CODESTARTobjDestruct(strm) - if(pThis->tOperationsMode == STREAMMODE_WRITE) + if(pThis->bAsyncWrite) + /* Note: mutex will be unlocked in stopWriter! */ + d_pthread_mutex_lock(&pThis->mut); + + if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); - /* ... then free resources */ + if(pThis->bAsyncWrite) { + stopWriter(pThis); + pthread_mutex_destroy(&pThis->mut); + pthread_cond_destroy(&pThis->notFull); + pthread_cond_destroy(&pThis->notEmpty); + pthread_cond_destroy(&pThis->isEmpty); + for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) { + free(pThis->asyncBuf[i].pBuf); + } + } else { + free(pThis->pIOBuf); + } + + /* Finally, we can free the resources. + * IMPORTANT: we MUST free this only AFTER the ansyncWriter has been stopped, else + * we get random errors... + */ if(pThis->fd != -1) strmCloseFile(pThis); - if(pThis->pszDir != NULL) - free(pThis->pszDir); - if(pThis->pIOBuf != NULL) - free(pThis->pIOBuf); - if(pThis->pszCurrFName != NULL) - free(pThis->pszCurrFName); - if(pThis->pszFName != NULL) - free(pThis->pszFName); + free(pThis->pszDir); + free(pThis->pZipBuf); + free(pThis->pszCurrFName); + free(pThis->pszFName); + ENDobjDestruct(strm) @@ -443,6 +728,9 @@ static rsRetVal strmCheckNextOutputFile(strm_t *pThis) if(pThis->fd == -1) FINALIZE; + /* wait for output to be empty, so that our counts are correct */ + strmWaitAsyncWriterDone(pThis); + if(pThis->iCurrOffs >= pThis->iMaxFileSize) { dbgoprint((obj_t*) pThis, "max file size %ld reached for %d, now %ld - starting new file\n", (long) pThis->iMaxFileSize, pThis->fd, (long) pThis->iCurrOffs); @@ -453,47 +741,375 @@ finalize_it: RETiRet; } + +/* try to recover a tty after a write error. This may have happend + * due to vhangup(), and, if so, we can simply re-open it. + */ +#ifdef linux +# define ERR_TTYHUP EIO +#else +# define ERR_TTYHUP EBADF +#endif +static rsRetVal +tryTTYRecover(strm_t *pThis, int err) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + if(err == ERR_TTYHUP) { + close(pThis->fd); + CHKiRet(doPhysOpen(pThis)); + } + +finalize_it: + RETiRet; +} +#undef ER_TTYHUP + + +/* issue write() api calls until either the buffer is completely + * written or an error occured (it may happen that multiple writes + * are required, what is perfectly legal. On exit, *pLenBuf contains + * the number of bytes actually written. + * rgerhards, 2009-06-08 + */ +static rsRetVal +doWriteCall(strm_t *pThis, uchar *pBuf, size_t *pLenBuf) +{ + ssize_t lenBuf; + ssize_t iTotalWritten; + ssize_t iWritten; + char *pWriteBuf; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + + lenBuf = *pLenBuf; + pWriteBuf = (char*) pBuf; + iTotalWritten = 0; + do { + iWritten = write(pThis->fd, pWriteBuf, lenBuf); + if(iWritten < 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("log file (%d) write error %d: %s\n", pThis->fd, err, errStr); + if(err == EINTR) { + /*NO ERROR, just continue */; + } else { + if(pThis->bIsTTY) { + CHKiRet(tryTTYRecover(pThis, err)); + } else { + ABORT_FINALIZE(RS_RET_IO_ERROR); + /* Would it make sense to cover more error cases? So far, I + * do not see good reason to do so. + */ + } + } + } + /* advance buffer to next write position */ + iTotalWritten += iWritten; + lenBuf -= iWritten; + pWriteBuf += iWritten; + } while(lenBuf > 0); /* Warning: do..while()! */ + + dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, (int) iWritten); + +finalize_it: + *pLenBuf = iTotalWritten; + RETiRet; +} + + + /* write memory buffer to a stream object. - * To support direct writes of large objects, this method may be called - * with a buffer pointing to some region other than the stream buffer itself. - * However, in that case the stream buffer must be empty (strmFlush() has to - * be called before), because we would otherwise mess up with the sequence - * inside the stream. -- rgerhards, 2008-01-10 */ -static rsRetVal strmWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static inline rsRetVal +doWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + DEFiRet; + + ASSERT(pThis != NULL); + + if(pThis->iZipLevel) { + CHKiRet(doZipWrite(pThis, pBuf, lenBuf)); + } else { + /* write without zipping */ + CHKiRet(strmPhysWrite(pThis, pBuf, lenBuf)); + } + +finalize_it: + RETiRet; +} + + +/* This function is called to "do" an async write call, what primarily means that + * the data is handed over to the writer thread (which will then do the actual write + * in parallel). Note that the stream mutex has already been locked by the + * strmWrite...() calls. Also note that we always have only a single producer, + * so we can simply serially assign the next free buffer to it and be sure that + * the very some producer comes back in sequence to submit the then-filled buffers. + * This also enables us to timout on partially written buffers. -- rgerhards, 2009-07-06 + */ +static inline rsRetVal +doAsyncWriteInternal(strm_t *pThis, size_t lenBuf) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + + while(pThis->iCnt >= STREAM_ASYNC_NUMBUFS) + d_pthread_cond_wait(&pThis->notFull, &pThis->mut); + + pThis->asyncBuf[pThis->iEnq % STREAM_ASYNC_NUMBUFS].lenBuf = lenBuf; + pThis->pIOBuf = pThis->asyncBuf[++pThis->iEnq % STREAM_ASYNC_NUMBUFS].pBuf; + + pThis->bDoTimedWait = 0; /* everything written, no need to timeout partial buffer writes */ + if(++pThis->iCnt == 1) + pthread_cond_signal(&pThis->notEmpty); + + RETiRet; +} + + +/* schedule writing to the stream. Depending on our concurrency settings, + * this either directly writes to the stream or schedules writing via + * the background thread. -- rgerhards, 2009-07-07 + */ +static rsRetVal +strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; - int iWritten; ASSERT(pThis != NULL); - ASSERT(pBuf == pThis->pIOBuf || pThis->iBufPtr == 0); + + if(pThis->bAsyncWrite) { + CHKiRet(doAsyncWriteInternal(pThis, lenBuf)); + } else { + CHKiRet(doWriteInternal(pThis, pBuf, lenBuf)); + } + + pThis->iBufPtr = 0; /* we are at the begin of a new buffer */ + +finalize_it: + RETiRet; +} + + + +/* This is the writer thread for asynchronous mode. + * -- rgerhards, 2009-07-06 + */ +static void* +asyncWriterThread(void *pPtr) +{ + int iDeq; + struct timespec t; + sbool bTimedOut = 0; + strm_t *pThis = (strm_t*) pPtr; + ISOBJ_TYPE_assert(pThis, strm); + + BEGINfunc +# if HAVE_PRCTL && defined PR_SET_NAME + if(prctl(PR_SET_NAME, "rs:asyn strmwr", 0, 0, 0) != 0) { + DBGPRINTF("prctl failed, not setting thread name for '%s'\n", "stream writer"); + } +#endif + + while(1) { /* loop broken inside */ + d_pthread_mutex_lock(&pThis->mut); + while(pThis->iCnt == 0) { + if(pThis->bStopWriter) { + pthread_cond_broadcast(&pThis->isEmpty); + d_pthread_mutex_unlock(&pThis->mut); + goto finalize_it; /* break main loop */ + } + if(bTimedOut && pThis->iBufPtr > 0) { + /* if we timed out, we need to flush pending data */ + strmFlush(pThis); + bTimedOut = 0; + continue; /* now we should have data */ + } + bTimedOut = 0; + timeoutComp(&t, pThis->iFlushInterval * 2000); /* *1000 millisconds */ + if(pThis->bDoTimedWait) { + if(pthread_cond_timedwait(&pThis->notEmpty, &pThis->mut, &t) != 0) { + int err = errno; + if(err == ETIMEDOUT) { + bTimedOut = 1; + } else { + bTimedOut = 1; + char errStr[1024]; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("stream async writer timeout with error (%d): %s - ignoring\n", + err, errStr); + } + } + } else { + d_pthread_cond_wait(&pThis->notEmpty, &pThis->mut); + } + } + + bTimedOut = 0; /* we may have timed out, but there *is* work to do... */ + + iDeq = pThis->iDeq++ % STREAM_ASYNC_NUMBUFS; + doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf); + // TODO: error check????? 2009-07-06 + + --pThis->iCnt; + if(pThis->iCnt < STREAM_ASYNC_NUMBUFS) { + pthread_cond_signal(&pThis->notFull); + if(pThis->iCnt == 0) + pthread_cond_broadcast(&pThis->isEmpty); + } + d_pthread_mutex_unlock(&pThis->mut); + } + +finalize_it: + ENDfunc + return NULL; /* to keep pthreads happy */ +} + + +/* sync the file to disk, so that any unwritten data is persisted. This + * also syncs the directory and thus makes sure that the file survives + * fatal failure. Note that we do NOT return an error status if the + * sync fails. Doing so would probably cause more trouble than it + * is worth (read: data loss may occur where we otherwise might not + * have it). -- rgerhards, 2009-06-08 + */ +#undef SYNCCALL +#if HAVE_FDATASYNC +# define SYNCCALL(x) fdatasync(x) +#else +# define SYNCCALL(x) fsync(x) +#endif +static rsRetVal +syncFile(strm_t *pThis) +{ + int ret; + DEFiRet; + + if(pThis->bIsTTY) + FINALIZE; /* TTYs can not be synced */ + + DBGPRINTF("syncing file %d\n", pThis->fd); + ret = SYNCCALL(pThis->fd); + if(ret != 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("sync failed for file %d with error (%d): %s - ignoring\n", + pThis->fd, err, errStr); + } + + if(pThis->fdDir != -1) { + ret = fsync(pThis->fdDir); + } + +finalize_it: + RETiRet; +} +#undef SYNCCALL + +/* physically write to the output file. the provided data is ready for + * writing (e.g. zipped if we are requested to do that). + * Note that if the write() API fails, we do not reset any pointers, but return + * an error code. That means we may redo work in the next iteration. + * rgerhards, 2009-06-04 + */ +static rsRetVal +strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + size_t iWritten; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); - iWritten = write(pThis->fd, pBuf, lenBuf); - dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, iWritten); - /* TODO: handle error case -- rgerhards, 2008-01-07 */ - - /* Now indicate buffer empty again. We do this in any case, because there - * is no way we could react more intelligently to an error during write. - * This MUST be done BEFORE strCheckNextOutputFile(), otherwise we have an - * endless loop. We reset the buffer pointer also in finalize_it - this is - * necessary if we run into problems. Not resetting it would again cause an - * endless loop. So it is better to loose some data (which also justifies - * duplicating that code, too...) -- rgerhards, 2008-01-10 - */ - pThis->iBufPtr = 0; + iWritten = lenBuf; + CHKiRet(doWriteCall(pThis, pBuf, &iWritten)); + pThis->iCurrOffs += iWritten; /* update user counter, if provided */ if(pThis->pUsrWCntr != NULL) *pThis->pUsrWCntr += iWritten; - if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) + if(pThis->bSync) { + CHKiRet(syncFile(pThis)); + } + + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { CHKiRet(strmCheckNextOutputFile(pThis)); + } else if(pThis->iSizeLimit != 0) { + CHKiRet(doSizeLimitProcessing(pThis)); + } finalize_it: - pThis->iBufPtr = 0; /* see comment above */ + RETiRet; +} + + +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + * Note that we always do a full deflateInit ... deflate ... deflateEnd + * sequence. While this is not optimal, we need to do it because we need + * to ensure that the file is readable even when we are aborted. Doing the + * full sequence brings us as far towards this goal as possible (and not + * doing it would be a total failure). It may be worth considering to + * add a config switch so that the user can decide the risk he is ready + * to take, but so far this is not yet implemented (not even requested ;)). + * rgerhards, 2009-06-04 + * For the time being, we take a very conservative approach and do not run this + * method multithreaded. This is done in an effort to solve a segfault condition + * that seems to be related to the zip code. -- rgerhards, 2009-09-22 + * TODO: make multithreaded again! + */ +static rsRetVal +doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + z_stream zstrm; + int zRet; /* zlib return state */ + sbool bzInitDone = FALSE; + DEFiRet; + assert(pThis != NULL); + assert(pBuf != NULL); + + /* allocate deflate state */ + zstrm.zalloc = Z_NULL; + zstrm.zfree = Z_NULL; + zstrm.opaque = Z_NULL; + zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateInit2()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + bzInitDone = TRUE; + + /* now doing the compression */ + zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ + zstrm.avail_in = lenBuf; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); + zstrm.avail_out = pThis->sIOBufSize; + zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + if(zstrm.avail_out == pThis->sIOBufSize) + break; /* this is valid, indicates end of compression --> see zlib howto */ + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, pThis->sIOBufSize - zstrm.avail_out)); + } while (zstrm.avail_out == 0); + assert(zstrm.avail_in == 0); /* all input will be used */ + +finalize_it: + if(bzInitDone) { + zRet = zlibw.DeflateEnd(&zstrm); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); + } + } RETiRet; } @@ -503,15 +1119,16 @@ finalize_it: * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 */ -rsRetVal strmFlush(strm_t *pThis) +static rsRetVal +strmFlush(strm_t *pThis) { DEFiRet; ASSERT(pThis != NULL); dbgoprint((obj_t*) pThis, "file %d flush, buflen %ld\n", pThis->fd, (long) pThis->iBufPtr); - if(pThis->tOperationsMode == STREAMMODE_WRITE && pThis->iBufPtr > 0) { - iRet = strmWriteInternal(pThis, pThis->pIOBuf, pThis->iBufPtr); + if(pThis->tOperationsMode != STREAMMODE_READ && pThis->iBufPtr > 0) { + iRet = strmSchedWrite(pThis, pThis->pIOBuf, pThis->iBufPtr); } RETiRet; @@ -545,7 +1162,7 @@ static rsRetVal strmSeek(strm_t *pThis, off_t offs) /* seek to current offset. This is primarily a helper to readjust the OS file * pointer after a strm object has been deserialized. */ -rsRetVal strmSeekCurrOffs(strm_t *pThis) +static rsRetVal strmSeekCurrOffs(strm_t *pThis) { DEFiRet; @@ -558,12 +1175,18 @@ rsRetVal strmSeekCurrOffs(strm_t *pThis) /* write a *single* character to a stream object -- rgerhards, 2008-01-10 */ -rsRetVal strmWriteChar(strm_t *pThis, uchar c) +static rsRetVal strmWriteChar(strm_t *pThis, uchar c) { DEFiRet; ASSERT(pThis != NULL); + if(pThis->bAsyncWrite) + d_pthread_mutex_lock(&pThis->mut); + + if(pThis->bDisabled) + ABORT_FINALIZE(RS_RET_STREAM_DISABLED); + /* if the buffer is full, we need to flush before we can write */ if(pThis->iBufPtr == pThis->sIOBufSize) { CHKiRet(strmFlush(pThis)); @@ -573,12 +1196,19 @@ rsRetVal strmWriteChar(strm_t *pThis, uchar c) pThis->iBufPtr++; finalize_it: + if(pThis->bAsyncWrite) + d_pthread_mutex_unlock(&pThis->mut); + RETiRet; } -/* write an integer value (actually a long) to a stream object */ -rsRetVal strmWriteLong(strm_t *pThis, long i) +/* write an integer value (actually a long) to a stream object + * Note that we do not need to lock the mutex here, because we call + * strmWrite(), which does the lock (aka: we must not lock it, else we + * would run into a recursive lock, resulting in a deadlock!) + */ +static rsRetVal strmWriteLong(strm_t *pThis, long i) { DEFiRet; uchar szBuf[32]; @@ -593,45 +1223,67 @@ finalize_it: } -/* write memory buffer to a stream object +/* write memory buffer to a stream object. + * process the data in chunks and copy it over to our buffer. The caller-provided data + * may theoritically be larger than our buffer. In that case, we do multiple copies. One + * may argue if it were more efficient to write out the caller-provided buffer in that case + * and earlier versions of rsyslog did this. However, this introduces a lot of complexity + * inside the buffered writer and potential performance bottlenecks when trying to solve + * it. Now keep in mind that we actually do (almost?) never have a case where the + * caller-provided buffer is larger than our one. So instead of optimizing a case + * which normally does not exist, we expect some degradation in its case but make us + * perform better in the regular cases. -- rgerhards, 2009-07-07 */ -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static rsRetVal +strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; - size_t iPartial; + size_t iWrite; + size_t iOffset; ASSERT(pThis != NULL); ASSERT(pBuf != NULL); - /* check if the to-be-written data is larger than our buffer size */ - if(lenBuf >= pThis->sIOBufSize) { - /* it is - so we do a direct write, that is most efficient. - * TODO: is it really? think about disk block sizes! - */ - CHKiRet(strmFlush(pThis)); /* we need to flush first!!! */ - CHKiRet(strmWriteInternal(pThis, pBuf, lenBuf)); - } else { - /* data fits into a buffer - we just need to see if it - * fits into the current buffer... - */ - if(pThis->iBufPtr + lenBuf > pThis->sIOBufSize) { - /* nope, so we must split it */ - iPartial = pThis->sIOBufSize - pThis->iBufPtr; /* this fits in current buf */ - if(iPartial > 0) { /* the buffer was exactly full, can not write anything! */ - memcpy(pThis->pIOBuf + pThis->iBufPtr, pBuf, iPartial); - pThis->iBufPtr += iPartial; - } +//DBGPRINTF("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n", pThis, pBuf,lenBuf, pThis->bDisabled, pThis->iSizeLimit, pThis->iCurrOffs); + if(pThis->bAsyncWrite) + d_pthread_mutex_lock(&pThis->mut); + + if(pThis->bDisabled) + ABORT_FINALIZE(RS_RET_STREAM_DISABLED); + + iOffset = 0; + do { + if(pThis->iBufPtr == pThis->sIOBufSize) { CHKiRet(strmFlush(pThis)); /* get a new buffer for rest of data */ - memcpy(pThis->pIOBuf, pBuf + iPartial, lenBuf - iPartial); - pThis->iBufPtr = lenBuf - iPartial; - } else { - /* we have space, so we simply copy over the string */ - memcpy(pThis->pIOBuf + pThis->iBufPtr, pBuf, lenBuf); - pThis->iBufPtr += lenBuf; } + iWrite = pThis->sIOBufSize - pThis->iBufPtr; /* this fits in current buf */ + if(iWrite > lenBuf) + iWrite = lenBuf; + memcpy(pThis->pIOBuf + pThis->iBufPtr, pBuf + iOffset, iWrite); + pThis->iBufPtr += iWrite; + iOffset += iWrite; + lenBuf -= iWrite; + } while(lenBuf > 0); + + /* now check if the buffer right at the end of the write is full and, if so, + * write it. This seems more natural than waiting (hours?) for the next message... + */ + if(pThis->iBufPtr == pThis->sIOBufSize) { + CHKiRet(strmFlush(pThis)); /* get a new buffer for rest of data */ } finalize_it: + if(pThis->bAsyncWrite) { + if(pThis->bDoTimedWait == 0) { + /* we potentially have a partial buffer, so re-activate the + * writer thread that it can set and pick up timeouts. + */ + pThis->bDoTimedWait = 1; + pthread_cond_signal(&pThis->notEmpty); + } + d_pthread_mutex_unlock(&pThis->mut); + } + RETiRet; } @@ -644,34 +1296,27 @@ DEFpropSetMeth(strm, iFileNumDigits, int) DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) - -rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) +DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, bSync, int) +DEFpropSetMeth(strm, sIOBufSize, size_t) +DEFpropSetMeth(strm, iSizeLimit, off_t) +DEFpropSetMeth(strm, iFlushInterval, int) +DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) + +static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { pThis->iMaxFiles = iNewVal; pThis->iFileNumDigits = getNumberDigits(iNewVal); return RS_RET_OK; } -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) -{ - DEFiRet; - - if(iNewVal & O_APPEND) - ABORT_FINALIZE(RS_RET_PARAM_ERROR); - - pThis->iAddtlOpenFlags = iNewVal; - -finalize_it: - RETiRet; -} - /* set the stream's file prefix * The passed-in string is duplicated. So if the caller does not need * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) { DEFiRet; @@ -685,7 +1330,7 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) if(pThis->pszFName != NULL) free(pThis->pszFName); - if((pThis->pszFName = malloc(sizeof(uchar) * iLenName + 1)) == NULL) + if((pThis->pszFName = MALLOC(sizeof(uchar) * (iLenName + 1))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */ @@ -701,7 +1346,7 @@ finalize_it: * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) { DEFiRet; @@ -712,7 +1357,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - if((pThis->pszDir = malloc(sizeof(uchar) * iLenDir + 1)) == NULL) + if((pThis->pszDir = MALLOC(sizeof(uchar) * iLenDir + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDir, pszDir, iLenDir + 1); /* always think about the \0! */ @@ -745,7 +1390,7 @@ finalize_it: * * rgerhards, 2008-01-10 */ -rsRetVal strmRecordBegin(strm_t *pThis) +static rsRetVal strmRecordBegin(strm_t *pThis) { ASSERT(pThis != NULL); ASSERT(pThis->bInRecord == 0); @@ -753,7 +1398,7 @@ rsRetVal strmRecordBegin(strm_t *pThis) return RS_RET_OK; } -rsRetVal strmRecordEnd(strm_t *pThis) +static rsRetVal strmRecordEnd(strm_t *pThis) { DEFiRet; ASSERT(pThis != NULL); @@ -775,7 +1420,7 @@ rsRetVal strmRecordEnd(strm_t *pThis) * We do not serialize the dynamic properties. * rgerhards, 2008-01-10 */ -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) +static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) { DEFiRet; int i; @@ -811,6 +1456,46 @@ finalize_it: } +/* duplicate a stream object excluding dynamic properties. This function is + * primarily meant to provide a duplicate that later on can be used to access + * the data. This is needed, for example, for a restart of the disk queue. + * Note that ConstructFinalize() is NOT called. So our caller may change some + * properties before finalizing things. + * rgerhards, 2009-05-26 + */ +rsRetVal +strmDup(strm_t *pThis, strm_t **ppNew) +{ + strm_t *pNew = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + assert(ppNew != NULL); + + CHKiRet(strmConstruct(&pNew)); + pNew->sType = pThis->sType; + pNew->iCurrFNum = pThis->iCurrFNum; + CHKmalloc(pNew->pszFName = ustrdup(pThis->pszFName)); + pNew->lenFName = pThis->lenFName; + CHKmalloc(pNew->pszDir = ustrdup(pThis->pszDir)); + pNew->lenDir = pThis->lenDir; + pNew->tOperationsMode = pThis->tOperationsMode; + pNew->tOpenMode = pThis->tOpenMode; + pNew->iMaxFileSize = pThis->iMaxFileSize; + pNew->iMaxFiles = pThis->iMaxFiles; + pNew->iFileNumDigits = pThis->iFileNumDigits; + pNew->bDeleteOnClose = pThis->bDeleteOnClose; + pNew->iCurrOffs = pThis->iCurrOffs; + + *ppNew = pNew; + pNew = NULL; + +finalize_it: + if(pNew != NULL) + strmDestruct(&pNew); + + RETiRet; +} /* set a user write-counter. This counter is initialized to zero and * receives the number of bytes written. It is accurate only after a @@ -821,7 +1506,7 @@ finalize_it: * any new set overwrites the previous one. * rgerhards, 2008-02-27 */ -rsRetVal +static rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt) { DEFiRet; @@ -841,8 +1526,8 @@ strmSetWCntr(strm_t *pThis, number_t *pWCnt) /* This function can be used as a generic way to set properties. * rgerhards, 2008-01-11 */ -#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) -rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) +#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, UCHAR_CONSTANT(name), sizeof(name) - 1) +static rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) { DEFiRet; @@ -881,7 +1566,7 @@ finalize_it: * reported on the second call may actually be lower than on the first call. This is due to * file circulation. A caller must deal with that. -- rgerhards, 2008-01-30 */ -rsRetVal +static rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs) { DEFiRet; @@ -909,8 +1594,39 @@ CODESTARTobjQueryInterface(strm) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - /*xxxpIf->oID = OBJvm; SAMPLE */ - + pIf->Construct = strmConstruct; + pIf->ConstructFinalize = strmConstructFinalize; + pIf->Destruct = strmDestruct; + pIf->ReadChar = strmReadChar; + pIf->UnreadChar = strmUnreadChar; + pIf->ReadLine = strmReadLine; + pIf->SeekCurrOffs = strmSeekCurrOffs; + pIf->Write = strmWrite; + pIf->WriteChar = strmWriteChar; + pIf->WriteLong = strmWriteLong; + pIf->SetFName = strmSetFName; + pIf->SetDir = strmSetDir; + pIf->Flush = strmFlush; + pIf->RecordBegin = strmRecordBegin; + pIf->RecordEnd = strmRecordEnd; + pIf->Serialize = strmSerialize; + pIf->GetCurrOffset = strmGetCurrOffset; + pIf->Dup = strmDup; + pIf->SetWCntr = strmSetWCntr; + /* set methods */ + pIf->SetbDeleteOnClose = strmSetbDeleteOnClose; + pIf->SetiMaxFileSize = strmSetiMaxFileSize; + pIf->SetiMaxFiles = strmSetiMaxFiles; + pIf->SetiFileNumDigits = strmSetiFileNumDigits; + pIf->SettOperationsMode = strmSettOperationsMode; + pIf->SettOpenMode = strmSettOpenMode; + pIf->SetsType = strmSetsType; + pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetbSync = strmSetbSync; + pIf->SetsIOBufSize = strmSetsIOBufSize; + pIf->SetiSizeLimit = strmSetiSizeLimit; + pIf->SetiFlushInterval = strmSetiFlushInterval; + pIf->SetpszSizeLimitCmd = strmSetpszSizeLimitCmd; finalize_it: ENDobjQueryInterface(strm) @@ -927,7 +1643,5 @@ BEGINObjClassInit(strm, 1, OBJ_IS_CORE_MODULE) OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmConstructFinalize); ENDObjClassInit(strm) - -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stream.h b/runtime/stream.h index 371358ab..5e3324c5 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -19,7 +19,29 @@ * can easily be persistet. The bottom line is that it makes much sense to * use this class whereever possible as its features may grow in the future. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * An important note on writing gzip format via zlib (kept anonymous + * by request): + * + * -------------------------------------------------------------------------- + * We'd like to make sure the output file is in full gzip format + * (compatible with gzip -d/zcat etc). There is a flag in how the output + * is initialized within zlib to properly add the gzip wrappers to the + * output. (gzip is effectively a small metadata wrapper around raw + * zstream output.) + * + * I had written an old bit of code to do this - the documentation on + * deflatInit2() was pretty tricky to nail down on this specific feature: + * + * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, + * int memLevel, int strategy); + * + * I believe "31" would be the value for the "windowBits" field that you'd + * want to try: + * + * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + * -------------------------------------------------------------------------- + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -47,20 +69,26 @@ #include "obj-types.h" #include "glbl.h" #include "stream.h" +#include "zlibw.h" +#include "apc.h" /* stream types */ typedef enum { STREAMTYPE_FILE_SINGLE = 0, /**< read a single file */ STREAMTYPE_FILE_CIRCULAR = 1, /**< circular files */ - STREAMTYPE_FILE_MONITOR = 2 /**< monitor a (third-party) file */ + STREAMTYPE_FILE_MONITOR = 2, /**< monitor a (third-party) file */ + STREAMTYPE_NAMED_PIPE = 3 /**< file is a named pipe (so far, tested for output only) */ } strmType_t; -typedef enum { +typedef enum { /* when extending, do NOT change existing modes! */ STREAMMMODE_INVALID = 0, STREAMMODE_READ = 1, - STREAMMODE_WRITE = 2 + STREAMMODE_WRITE = 2, + STREAMMODE_WRITE_TRUNC = 3, + STREAMMODE_WRITE_APPEND = 4 } strmMode_t; +#define STREAM_ASYNC_NUMBUFS 2 /* must be a power of 2 -- TODO: make configurable */ /* The strm_t data structure */ typedef struct strm_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ @@ -71,61 +99,96 @@ typedef struct strm_s { int lenFName; strmMode_t tOperationsMode; mode_t tOpenMode; - int iAddtlOpenFlags; /* can be used to specifiy additional (compatible!) open flags */ int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ - int bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ + sbool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ - size_t sIOBufSize;/* size of IO buffer */ + sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ + sbool bSync; /* sync this file after every write? */ + size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; int fd; /* the file descriptor, -1 if closed */ + int fdDir; /* the directory's descriptor, in case bSync is requested (-1 if closed) */ uchar *pszCurrFName; /* name of current file (if open) */ - uchar *pIOBuf; /* io Buffer */ + uchar *pIOBuf; /* the iobuffer currently in use to gather data */ size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ - int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + sbool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + sbool bInClose; /* used to break "deadly close loops", tells us we are already inside a close */ + int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ + Bytef *pZipBuf; + /* support for async flush procesing */ + sbool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ + sbool bStopWriter; /* shall writer thread terminate? */ + sbool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ + int iFlushInterval; /* flush in which interval - 0, no flushing */ + apc_id_t apcID; /* id of current Apc request (used for cancelling) */ + pthread_mutex_t mut;/* mutex for flush in async mode */ + pthread_cond_t notFull; + pthread_cond_t notEmpty; + pthread_cond_t isEmpty; + unsigned short iEnq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ + unsigned short iDeq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ + short iCnt; /* current nbr of elements in buffer */ + struct { + uchar *pBuf; + size_t lenBuf; + } asyncBuf[STREAM_ASYNC_NUMBUFS]; + pthread_t writerThreadID; + int apcRequested; /* is an apc Requested? */ + /* support for omfile size-limiting commands, special counters, NOT persisted! */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ + sbool bIsTTY; /* is this a tty file? */ } strm_t; + /* interfaces */ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(strm_t **ppThis); + rsRetVal (*ConstructFinalize)(strm_t *pThis); + rsRetVal (*Destruct)(strm_t **ppThis); + rsRetVal (*SetMaxFileSize)(strm_t *pThis, int64 iMaxFileSize); + rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); + rsRetVal (*ReadChar)(strm_t *pThis, uchar *pC); + rsRetVal (*UnreadChar)(strm_t *pThis, uchar c); + rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr); + rsRetVal (*SeekCurrOffs)(strm_t *pThis); + rsRetVal (*Write)(strm_t *pThis, uchar *pBuf, size_t lenBuf); + rsRetVal (*WriteChar)(strm_t *pThis, uchar c); + rsRetVal (*WriteLong)(strm_t *pThis, long i); + rsRetVal (*SetFName)(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); + rsRetVal (*SetDir)(strm_t *pThis, uchar *pszDir, size_t iLenDir); + rsRetVal (*Flush)(strm_t *pThis); + rsRetVal (*RecordBegin)(strm_t *pThis); + rsRetVal (*RecordEnd)(strm_t *pThis); + rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm); + rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); + rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); + rsRetVal (*Dup)(strm_t *pThis, strm_t **ppNew); + INTERFACEpropSetMeth(strm, bDeleteOnClose, int); + INTERFACEpropSetMeth(strm, iMaxFileSize, int); + INTERFACEpropSetMeth(strm, iMaxFiles, int); + INTERFACEpropSetMeth(strm, iFileNumDigits, int); + INTERFACEpropSetMeth(strm, tOperationsMode, int); + INTERFACEpropSetMeth(strm, tOpenMode, mode_t); + INTERFACEpropSetMeth(strm, sType, strmType_t); + INTERFACEpropSetMeth(strm, iZipLevel, int); + INTERFACEpropSetMeth(strm, bSync, int); + INTERFACEpropSetMeth(strm, sIOBufSize, size_t); + INTERFACEpropSetMeth(strm, iSizeLimit, off_t); + INTERFACEpropSetMeth(strm, iFlushInterval, int); + INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); ENDinterface(strm) -#define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ /* prototypes */ -rsRetVal strmConstruct(strm_t **ppThis); -rsRetVal strmConstructFinalize(strm_t __attribute__((unused)) *pThis); -rsRetVal strmDestruct(strm_t **ppThis); -rsRetVal strmSetMaxFileSize(strm_t *pThis, int64 iMaxFileSize); -rsRetVal strmSetFileName(strm_t *pThis, uchar *pszName, size_t iLenName); -rsRetVal strmReadChar(strm_t *pThis, uchar *pC); -rsRetVal strmUnreadChar(strm_t *pThis, uchar c); -rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr); -rsRetVal strmSeekCurrOffs(strm_t *pThis); -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); -rsRetVal strmWriteChar(strm_t *pThis, uchar c); -rsRetVal strmWriteLong(strm_t *pThis, long i); -rsRetVal strmSetFName(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); -rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir); -rsRetVal strmFlush(strm_t *pThis); -rsRetVal strmRecordBegin(strm_t *pThis); -rsRetVal strmRecordEnd(strm_t *pThis); -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm); -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal); -rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs); -rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt); PROTOTYPEObjClassInit(strm); -PROTOTYPEpropSetMeth(strm, bDeleteOnClose, int); -PROTOTYPEpropSetMeth(strm, iMaxFileSize, int); -PROTOTYPEpropSetMeth(strm, iMaxFiles, int); -PROTOTYPEpropSetMeth(strm, iFileNumDigits, int); -PROTOTYPEpropSetMeth(strm, tOperationsMode, int); -PROTOTYPEpropSetMeth(strm, tOpenMode, mode_t); -PROTOTYPEpropSetMeth(strm, sType, strmType_t); #endif /* #ifndef STREAM_H_INCLUDED */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 07256fab..ccf115c1 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -6,8 +6,9 @@ * Please see syslogd.c for license information. * All functions in this "class" start with rsCStr (rsyslog Counted String). * begun 2005-09-07 rgerhards + * did some optimization (read: bugs!) rgerhards, 2009-06-16 * - * Copyright (C) 2007-2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2007-2009 by Rainer Gerhards and Adiscon GmbH * * This file is part of the rsyslog runtime library. * @@ -40,6 +41,7 @@ #include "regexp.h" #include "obj.h" +uchar* rsCStrGetSzStr(cstr_t *pThis); /* ################################################################# * * private members * @@ -54,22 +56,20 @@ DEFobjCurrIf(regexp) * ################################################################# */ -rsRetVal rsCStrConstruct(cstr_t **ppThis) +rsRetVal cstrConstruct(cstr_t **ppThis) { DEFiRet; cstr_t *pThis; ASSERT(ppThis != NULL); - if((pThis = (cstr_t*) calloc(1, sizeof(cstr_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pThis = (cstr_t*) calloc(1, sizeof(cstr_t))); rsSETOBJTYPE(pThis, OIDrsCStr); pThis->pBuf = NULL; pThis->pszBuf = NULL; pThis->iBufSize = 0; pThis->iStrLen = 0; - pThis->iAllocIncrement = RS_STRINGBUF_ALLOC_INCREMENT; *ppThis = pThis; finalize_it: @@ -89,8 +89,8 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); - pThis->iBufSize = pThis->iStrLen = strlen((char*)(char *) sz); - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -119,7 +119,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = pFrom->iStrLen; - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -137,24 +137,8 @@ void rsCStrDestruct(cstr_t **ppThis) { cstr_t *pThis = *ppThis; - /* rgerhards 2005-10-19: The free of pBuf was contained in conditional compilation. - * The code was only compiled if STRINGBUF_TRIM_ALLOCSIZE was set to 1. I honestly - * do not know why it was so, I think it was an artifact. Anyhow, I have changed this - * now. Should there any issue occur, this comment hopefully will shed some light - * on what happened. I re-verified, and this function has never before been called - * by anyone. So changing it can have no impact for obvious reasons... - * - * rgerhards, 2008-02-20: I changed the interface to the new calling conventions, where - * the destructor receives a pointer to the object, so that it can set it to NULL. - */ - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } - - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - } - + free(pThis->pBuf); + free(pThis->pszBuf); RSFREEOBJ(pThis); *ppThis = NULL; } @@ -166,36 +150,32 @@ void rsCStrDestruct(cstr_t **ppThis) * allocated. In practice, a bit more is allocated because we envision that * some more characters may be added after these. * rgerhards, 2008-01-07 + * changed to utilized realloc() -- rgerhards, 2009-06-16 */ -static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) +rsRetVal +rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { - DEFiRet; uchar *pNewBuf; - size_t iNewSize; + unsigned short iNewSize; + DEFiRet; /* first compute the new size needed */ - if(iMinNeeded > pThis->iAllocIncrement) { - /* we allocate "n" iAllocIncrements. Usually, that should + if(iMinNeeded > RS_STRINGBUF_ALLOC_INCREMENT) { + /* we allocate "n" ALLOC_INCREMENTs. Usually, that should * leave some room after the absolutely needed one. It also * reduces memory fragmentation. Note that all of this are * integer operations (very important to understand what is * going on)! Parenthesis are for better readibility. */ - iNewSize = ((iMinNeeded / pThis->iAllocIncrement) + 1) * pThis->iAllocIncrement; + iNewSize = (iMinNeeded / RS_STRINGBUF_ALLOC_INCREMENT + 1) * RS_STRINGBUF_ALLOC_INCREMENT; } else { - iNewSize = pThis->iBufSize + pThis->iAllocIncrement; + iNewSize = pThis->iBufSize + RS_STRINGBUF_ALLOC_INCREMENT; } iNewSize += pThis->iBufSize; /* add current size */ - /* and then allocate and copy over */ /* DEV debugging only: dbgprintf("extending string buffer, old %d, new %d\n", pThis->iBufSize, iNewSize); */ - if((pNewBuf = (uchar*) malloc(iNewSize * sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - memcpy(pNewBuf, pThis->pBuf, pThis->iBufSize); + CHKmalloc(pNewBuf = (uchar*) realloc(pThis->pBuf, iNewSize * sizeof(uchar))); pThis->iBufSize = iNewSize; - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } pThis->pBuf = pNewBuf; finalize_it: @@ -243,7 +223,7 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz) /* append the contents of one cstr_t object to another * rgerhards, 2008-02-25 */ -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) { return rsCStrAppendStrWithLen(pThis, pstrAppend->pBuf, pstrAppend->iStrLen); } @@ -264,45 +244,18 @@ finalize_it: } -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c) -{ - DEFiRet; - - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(pThis->iStrLen >= pThis->iBufSize) { - CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ - } - - /* ok, when we reach this, we have sufficient memory */ - *(pThis->pBuf + pThis->iStrLen++) = c; - - /* check if we need to invalidate an sz representation! */ - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - pThis->pszBuf = NULL; - } - -finalize_it: - RETiRet; -} - - /* Sets the string object to the classigal sz-string provided. * Any previously stored vlaue is discarded. If a NULL pointer * the the new value (pszNew) is provided, an empty string is - * created (this is NOT an error!). Property iAllocIncrement is - * not modified by this function. + * created (this is NOT an error!). * rgerhards, 2005-10-18 */ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->pBuf != NULL) - free(pThis->pBuf); - if(pThis->pszBuf != NULL) - free(pThis->pszBuf); + free(pThis->pBuf); + free(pThis->pszBuf); if(pszNew == NULL) { pThis->iStrLen = 0; pThis->iBufSize = 0; @@ -312,10 +265,9 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) pThis->iStrLen = strlen((char*)pszNew); pThis->iBufSize = pThis->iStrLen; pThis->pszBuf = NULL; - /* iAllocIncrement is NOT modified! */ /* now save the new value */ - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); return RS_RET_OUT_OF_MEMORY; } @@ -363,7 +315,7 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) if(pThis->pBuf != NULL) if(pThis->pszBuf == NULL) { /* we do not yet have a usable sz version - so create it... */ - if((pThis->pszBuf = malloc((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { + if((pThis->pszBuf = MALLOC((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { /* TODO: think about what to do - so far, I have no bright * idea... rgerhards 2005-09-07 */ @@ -395,30 +347,18 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) * MUST be freed by the caller. The function might return NULL if * no memory can be allocated. * - * TODO: - * This function should at some time become special. The base idea is to - * add one extra byte to the end of the regular buffer, so that we can - * convert it to an szString without the need to copy. The extra memory - * footprint is not hefty, but the performance gain is potentially large. - * To get it done now, I am not doing the optimiziation right now. - * rgerhards, 2005-09-07 + * This is the NEW replacement for rsCStrConvSzStrAndDestruct which does + * no longer utilize a special buffer but soley works on pBuf (and also + * assumes that cstrFinalize had been called). * - * rgerhards, 2007-09-04: I have changed the interface of this function. It now - * returns an rsRetVal, so that we can communicate back if we have an error. - * Using the standard method is much better than returning NULL. Secondly, NULL - * was not actually an error - it was in indication if the string was empty. - * This was needed in some parts of the code, in others not. I have now added - * a second parameter to specify what the caller needs. I hope these changes - * will make it less likely that the function is called incorrectly, what - * previously happend quite often and was the cause of a number of program - * aborts. So the parameters are now: + * Parameters are as follows: * pointer to the object, pointer to string-pointer to receive string and * bRetNULL: 0 - must not return NULL on empty string, return "" in that * case, 1 - return NULL instead of an empty string. * PLEASE NOTE: the caller must free the memory returned in ppSz in any case * (except, of course, if it is NULL). */ -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) { DEFiRet; uchar* pRetBuf; @@ -429,14 +369,13 @@ rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - if((pRetBuf = malloc(sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pRetBuf = MALLOC(sizeof(uchar))); *pRetBuf = '\0'; } else { pRetBuf = NULL; } } else - pRetBuf = rsCStrGetSzStr(pThis); + pRetBuf = pThis->pBuf; *ppSz = pRetBuf; @@ -445,64 +384,11 @@ finalize_it: * that we can NOT use the rsCStrDestruct function as it would * also free the sz String buffer, which we pass on to the user. */ - if(pThis->pBuf != NULL) - free(pThis->pBuf); RSFREEOBJ(pThis); - RETiRet; } -#if STRINGBUF_TRIM_ALLOCSIZE == 1 - /* Only in this mode, we need to trim the string. To do - * so, we must allocate a new buffer of the exact - * string size, and then copy the old one over. - */ - /* WARNING - * STRINGBUF_TRIM_ALLOCSIZE can, in theory, be used to trim - * memory buffers. This part of the code was inherited from - * liblogging (where it is used in a different context) but - * never put to use in rsyslog. The reason is that it is hardly - * imaginable where the extra performance cost is worth the save - * in memory alloc. Then Anders Blomdel rightfully pointed out that - * the code does not work at all - and nobody even know that it - * probably shouldn't. Rather than removing, I deciced to somewhat - * fix the code, so that this feature may be enabled if somebody - * really has a need for it. Be warned, however, that I NEVER - * tested the fix. So if you intend to use this feature, you must - * do full testing before you rely on it. -- rgerhards, 2008-02-12 - */ -rsRetVal rsCStrFinish(cstr_t __attribute__((unused)) *pThis) -{ - DEFiRet; - uchar* pBuf; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if((pBuf = malloc((pThis->iStrLen) * sizeof(uchar))) == NULL) - { /* OK, in this case we use the previous buffer. At least - * we have it ;) - */ - } - else - { /* got the new buffer, so let's use it */ - memcpy(pBuf, pThis->pBuf, pThis->iStrLen); - pThis->pBuf = pBuf; - } - - RETiRet; -} -#endif /* #if STRINGBUF_TRIM_ALLOCSIZE == 1 */ - - -void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) -{ - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - assert(iNewIncrement > 0); - - pThis->iAllocIncrement = iNewIncrement; -} - - /* return the length of the current string * 2005-09-09 rgerhards * Please note: this is only a function in a debug build. @@ -510,7 +396,7 @@ void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) * This is due to performance reasons. */ #ifndef NDEBUG -int rsCStrLen(cstr_t *pThis) +int cstrLen(cstr_t *pThis) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); return(pThis->iStrLen); @@ -561,6 +447,27 @@ rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis) return RS_RET_OK; } +/* Trim trailing whitespace from a given string + */ +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis) +{ + register int i; + register uchar *pC; + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + i = pThis->iStrLen; + pC = pThis->pBuf + i - 1; + while(i > 0 && isspace((int)*pC)) { + --pC; + --i; + } + /* i now is the new string length! */ + pThis->iStrLen = i; + pThis->pBuf[pThis->iStrLen] = '0'; /* we always have this space */ + + return RS_RET_OK; +} + /* compare two string objects - works like strcmp(), but operates * on CStr objects. Please note that this version here is * faster in the majority of cases, simply because it can @@ -1027,56 +934,6 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz) } -#if 0 /* read comment below why this is commented out. In short: for future use! */ -/* locate the first occurence of a standard sz string inside a rsCStr object. - * Returns the offset (0-bound) of this first occurrence. If not found, -1 is - * returned. - * rgerhards 2005-09-19 - * WARNING: I accidently created this function (I later noticed I didn't relly - * need it... I will not remove the function, as it probably is useful - * some time later. However, it is not fully tested, so start with testing - * it before you put it to first use). - */ -int rsCStrLocateSzStr(cstr_t *pThis, uchar *sz) -{ - int iLenSz; - int i; - int iMax; - int bFound; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(sz == NULL) - return 0; - - iLenSz = strlen((char*)sz); - if(iLenSz == 0) - return 0; - - /* compute the largest index where a match could occur - after all, - * the to-be-located string must be able to be present in the - * searched string (it needs its size ;)). - */ - iMax = pThis->iStrLen - iLenSz; - - bFound = 0; - i = 0; - while(i < iMax && !bFound) { - int iCheck; - uchar *pComp = pThis->pBuf + i; - for(iCheck = 0 ; iCheck < iLenSz ; ++iCheck) - if(*(pComp + iCheck) != *(sz + iCheck)) - break; - if(iCheck == iLenSz) - bFound = 1; /* found! - else it wouldn't be equal */ - else - ++i; /* on to the next try */ - } - - return(bFound ? i : -1); -} -#endif /* end comment out */ - - /* our exit function. TODO: remove once converted to a class * rgerhards, 2008-03-11 */ @@ -1100,11 +957,5 @@ finalize_it: } -/* - * Local variables: - * c-indent-level: 8 - * c-basic-offset: 8 - * tab-width: 8 - * End: - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 684133bb..c5130238 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -1,5 +1,5 @@ -/*! \file stringbuf.h - * \brief The counted string object +/* stringbuf.h + * The counted string object * * This is the byte-counted string class for rsyslog. It is a replacement * for classical \0 terminated string functions. We introduce it in @@ -11,8 +11,7 @@ * \date 2005-09-07 * Initial version begun. * - * All functions in this "class" start with rsCStr (rsyslog Counted String). - * Copyright 2005 + * Copyright 2005-2009 * Rainer Gerhards and Adiscon GmbH. All Rights Reserved. * * This file is part of the rsyslog runtime library. @@ -36,6 +35,8 @@ #ifndef _STRINGBUF_H_INCLUDED__ #define _STRINGBUF_H_INCLUDED__ 1 +#include <assert.h> + /** * The dynamic string buffer object. */ @@ -48,14 +49,14 @@ typedef struct cstr_s uchar *pszBuf; /**< pointer to the sz version of the string (after it has been created )*/ size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ - size_t iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ } cstr_t; /** * Construct a rsCStr object. */ -rsRetVal rsCStrConstruct(cstr_t **ppThis); +rsRetVal cstrConstruct(cstr_t **ppThis); +#define rsCStrConstruct(x) cstrConstruct((x)) rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz); rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); @@ -63,14 +64,88 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); * Destruct the string buffer object. */ void rsCStrDestruct(cstr_t **ppThis); +#define cstrDestruct(x) rsCStrDestruct((x)) -/** - * Append a character to an existing string. If necessary, the - * method expands the string buffer. - * - * \param c Character to append to string. + +/* Append a character to the current string object. This may only be done until + * cstrFinalize() is called. + * rgerhards, 2009-06-16 + */ +rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded); /* our helper, NOT a public interface! */ +static inline rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) +{ + rsRetVal iRet = RS_RET_OK; + + if(pThis->iStrLen >= pThis->iBufSize) { + CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ + } + + /* ok, when we reach this, we have sufficient memory */ + *(pThis->pBuf + pThis->iStrLen++) = c; + +finalize_it: + return iRet; +} + + +/* some inline functions for things that are really frequently called... */ + +/* Finalize the string object. This must be called after all data is added to it + * but before that data is used. + * rgerhards, 2009-06-16 + */ +static inline rsRetVal +cstrFinalize(cstr_t *pThis) +{ + rsRetVal iRet = RS_RET_OK; + + if(pThis->iStrLen > 0) { + /* terminate string only if one exists */ + CHKiRet(cstrAppendChar(pThis, '\0')); + --pThis->iStrLen; /* do NOT count the \0 byte */ + } + +finalize_it: + return iRet; +} + + +/* Returns the cstr data as a classical C sz string. We use that the + * Finalizer did properly terminate our string (but we may stil be NULL). + * So it is vital that the finalizer is called BEFORe this function here! + * The caller must not free or otherwise manipulate the returned string and must not + * destroy the CStr object as long as the ascii string is used. + * This function may return NULL, if the string is currently NULL. This + * is a feature, not a bug. If you need non-NULL in any case, use + * cstrGetSzStrNoNULL() instead. + * Note that due to the new single-buffer interface this function almost does nothing! + * rgerhards, 2006-09-16 */ -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); +static inline uchar* cstrGetSzStr(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + return(pThis->pBuf); +} + + +/* Converts the CStr object to a classical sz string and returns that. + * Same restrictions as in cstrGetSzStr() applies (see there!). This + * function here guarantees that a valid string is returned, even if + * the CStr object currently holds a NULL pointer string buffer. If so, + * "" is returned. + * rgerhards 2005-10-19 + * WARNING: The returned pointer MUST NOT be freed, as it may be + * obtained from that constant memory pool (in case of NULL!) + */ +static inline uchar* cstrGetSzStrNoNULL(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + if(pThis->pBuf == NULL) + return (uchar*) ""; + else + return cstrGetSzStr(pThis); +} + /** * Truncate "n" number of characters from the end of the @@ -81,6 +156,7 @@ rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); rsRetVal rsCStrTruncate(cstr_t *pThis, size_t nTrunc); rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis); +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis); /** * Append a string to the buffer. For performance reasons, @@ -98,22 +174,6 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz); */ rsRetVal rsCStrAppendStrWithLen(cstr_t *pThis, uchar* psz, size_t iStrLen); -/** - * Set a new allocation incremet. This will influence - * the allocation the next time the string will be expanded. - * It can be set and changed at any time. If done immediately - * after custructing the StrB object, this will also be - * the inital allocation. - * - * \param iNewIncrement The new increment size - * - * \note It is possible to use a very low increment, e.g. 1 byte. - * This can generate a considerable overhead. We highly - * advise not to use an increment below 32 bytes, except - * if you are very well aware why you are doing it ;) - */ -void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement); -#define rsCStrGetAllocIncrement(pThis) ((pThis)->iAllocIncrement) /** * Append an integer to the string. No special formatting is @@ -123,10 +183,9 @@ rsRetVal rsCStrAppendInt(cstr_t *pThis, long i); rsRetVal strExit(void); /* TODO: remove once we have a real object interface! */ -uchar* rsCStrGetSzStr(cstr_t *pThis); +uchar* __attribute__((deprecated)) rsCStrGetSzStr(cstr_t *pThis); uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis); rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew); -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); int rsCStrCStrCmp(cstr_t *pCS1, cstr_t *pCS2); int rsCStrSzStrCmp(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrOffsetSzStrCmp(cstr_t *pCS1, size_t iOffset, uchar *psz, size_t iLenSz); @@ -140,27 +199,22 @@ rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *cache) void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); + +/* in migration */ +#define rsCStrAppendCStr(pThis, pstrAppend) cstrAppendCStr(pThis, pstrAppend) + +/* new calling interface */ +rsRetVal cstrFinalize(cstr_t *pThis); +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); /* now come inline-like functions */ #ifdef NDEBUG -# define rsCStrLen(x) ((int)((x)->iStrLen)) -#else - int rsCStrLen(cstr_t *pThis); -#endif - -#if STRINGBUF_TRIM_ALLOCSIZE != 1 -/* This is the normal case (see comment in rsCStrFinish!). In those cases, the function - * simply needs to do nothing, so that we can save us the function call. - * rgerhards, 2008-02-12 - */ -# define rsCStrFinish(pThis) RS_RET_OK +# define cstrLen(x) ((int)((x)->iStrLen)) #else - /** - * Finish the string buffer dynamic allocation. - */ - rsRetVal rsCStrFinish(cstr_t *pThis); + int cstrLen(cstr_t *pThis); #endif +#define rsCStrLen(s) cstrLen((s)) #define rsCStrGetBufBeg(x) ((x)->pBuf) diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c new file mode 100644 index 00000000..0aeebe03 --- /dev/null +++ b/runtime/strms_sess.c @@ -0,0 +1,300 @@ +/* strms_sess.c + * + * This implements a session of the strmsrv object. For general + * comments, see header of strmsrv.c. + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include <ctype.h> + +#include "rsyslog.h" +#include "dirty.h" +#include "module-template.h" +#include "net.h" +#include "strmsrv.h" +#include "strms_sess.h" +#include "obj.h" +#include "errmsg.h" +#include "netstrm.h" +#include "msg.h" +#include "datetime.h" + + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(datetime) + +static int iMaxLine; /* maximum size of a single message */ + +/* forward definitions */ +static rsRetVal Close(strms_sess_t *pThis); + + +/* Standard-Constructor */ +BEGINobjConstruct(strms_sess) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(strms_sess) + + +/* ConstructionFinalizer + */ +static rsRetVal +strms_sessConstructFinalize(strms_sess_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + if(pThis->pSrv->OnSessConstructFinalize != NULL) { + CHKiRet(pThis->pSrv->OnSessConstructFinalize(&pThis->pUsr)); + } + +finalize_it: + RETiRet; +} + + +/* destructor for the strms_sess object */ +BEGINobjDestruct(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strms_sess) + if(pThis->pStrm != NULL) + netstrm.Destruct(&pThis->pStrm); + + if(pThis->pSrv->pOnSessDestruct != NULL) { + pThis->pSrv->pOnSessDestruct(&pThis->pUsr); + } + /* now destruct our own properties */ + free(pThis->fromHost); + free(pThis->fromHostIP); +ENDobjDestruct(strms_sess) + + +/* debugprint for the strms_sess object */ +BEGINobjDebugPrint(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strms_sess) +ENDobjDebugPrint(strms_sess) + + +/* set property functions */ +/* set the hostname. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHost() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-04-24 + */ +static rsRetVal +SetHost(strms_sess_t *pThis, uchar *pszHost) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHost); + pThis->fromHost = pszHost; + RETiRet; +} + +/* set the remote host's IP. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHostIP() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-05-16 + */ +static rsRetVal +SetHostIP(strms_sess_t *pThis, uchar *pszHostIP) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHostIP); + pThis->fromHostIP = pszHostIP; + RETiRet; +} + +static rsRetVal +SetStrm(strms_sess_t *pThis, netstrm_t *pStrm) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + pThis->pStrm = pStrm; + RETiRet; +} + + +/* set our parent, the strmsrv object */ +static rsRetVal +SetStrmsrv(strms_sess_t *pThis, strmsrv_t *pSrv) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + ISOBJ_TYPE_assert(pSrv, strmsrv); + pThis->pSrv = pSrv; + RETiRet; +} + + +/* set our parent listener info*/ +static rsRetVal +SetLstnInfo(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pLstnInfo != NULL); + pThis->pLstnInfo = pLstnInfo; + RETiRet; +} + + +static rsRetVal +SetUsrP(strms_sess_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + + +static void * +GetUsrP(strms_sess_t *pThis) +{ + return pThis->pUsr; +} + + +/* Closes a STRM session + * No attention is paid to the return code + * of close, so potential-double closes are not detected. + */ +static rsRetVal +Close(strms_sess_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strms_sess); + netstrm.Destruct(&pThis->pStrm); + free(pThis->fromHost); + pThis->fromHost = NULL; /* not really needed, but... */ + free(pThis->fromHostIP); + pThis->fromHostIP = NULL; /* not really needed, but... */ + + RETiRet; +} + + + +/* Processes the data received via a STRM session. If there + * is no other way to handle it, data is discarded. + * Input parameter data is the data received, iLen is its + * len as returned from recv(). iLen must be 1 or more (that + * is errors must be handled by caller!). iSTRMSess must be + * the index of the STRM session that received the data. + * rgerhards 2005-07-04 + * And another change while generalizing. We now return either + * RS_RET_OK, which means the session should be kept open + * or anything else, which means it must be closed. + * rgerhards, 2008-03-01 + */ +static rsRetVal +DataRcvd(strms_sess_t *pThis, char *pData, size_t iLen) +{ + DEFiRet; + char *pEnd; + + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pData != NULL); + assert(iLen > 0); + + /* We now copy the message to the session buffer. */ + pEnd = pData + iLen; /* this is one off, which is intensional */ + + while(pData < pEnd) { + CHKiRet(pThis->pSrv->OnCharRcvd(pThis, (uchar)*pData++)); + } + +finalize_it: + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strms_sess) +CODESTARTobjQueryInterface(strms_sess) + if(pIf->ifVersion != strms_sessCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strms_sessDebugPrint; + pIf->Construct = strms_sessConstruct; + pIf->ConstructFinalize = strms_sessConstructFinalize; + pIf->Destruct = strms_sessDestruct; + + pIf->Close = Close; + pIf->DataRcvd = DataRcvd; + + pIf->SetUsrP = SetUsrP; + pIf->GetUsrP = GetUsrP; + pIf->SetStrmsrv = SetStrmsrv; + pIf->SetLstnInfo = SetLstnInfo; + pIf->SetHost = SetHost; + pIf->SetHostIP = SetHostIP; + pIf->SetStrm = SetStrm; +finalize_it: +ENDobjQueryInterface(strms_sess) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strms_sess, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strms_sess) + /* release objects we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(datetime, CORE_COMPONENT); +ENDObjClassExit(strms_sess) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strms_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + CHKiRet(objUse(glbl, CORE_COMPONENT)); + iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ + objRelease(glbl, CORE_COMPONENT); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strms_sessDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strms_sessConstructFinalize); +ENDObjClassInit(strms_sess) + +/* vim:set ai: + */ diff --git a/runtime/strms_sess.h b/runtime/strms_sess.h new file mode 100644 index 00000000..6483c0c3 --- /dev/null +++ b/runtime/strms_sess.h @@ -0,0 +1,75 @@ +/* Definitions for strms_sess class. This implements a session of the + * generic stream server. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMS_SESS_H +#define INCLUDED_STRMS_SESS_H + +#include "obj.h" + +/* a forward-definition, we are somewhat cyclic */ +struct strmsrv_s; + +/* the strms_sess object */ +struct strms_sess_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + strmsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ + strmLstnPortList_t *pLstnInfo; /* pointer back to listener info */ + netstrm_t *pStrm; +// uchar *pMsg; /* message (fragment) received */ + uchar *fromHost; + uchar *fromHostIP; + void *pUsr; /* a user-pointer */ +}; + + +/* interfaces */ +BEGINinterface(strms_sess) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strms_sess); + rsRetVal (*Construct)(strms_sess_t **ppThis); + rsRetVal (*ConstructFinalize)(strms_sess_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strms_sess_t **ppThis); + rsRetVal (*Close)(strms_sess_t *pThis); + rsRetVal (*DataRcvd)(strms_sess_t *pThis, char *pData, size_t iLen); + /* set methods */ + rsRetVal (*SetStrmsrv)(strms_sess_t *pThis, struct strmsrv_s *pSrv); + rsRetVal (*SetLstnInfo)(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo); + rsRetVal (*SetUsrP)(strms_sess_t*, void*); + void* (*GetUsrP)(strms_sess_t*); + rsRetVal (*SetHost)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetHostIP)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetStrm)(strms_sess_t *pThis, netstrm_t*); + rsRetVal (*SetOnMsgReceive)(strms_sess_t *pThis, rsRetVal (*OnMsgReceive)(strms_sess_t*, uchar*, int)); +ENDinterface(strms_sess) +#define strms_sessCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface changes + * to version v2, rgerhards, 2009-05-22 + * - Data structures changed + * - SetLstnInfo entry point added + */ + + +/* prototypes */ +PROTOTYPEObj(strms_sess); + + +#endif /* #ifndef INCLUDED_STRMS_SESS_H */ diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c new file mode 100644 index 00000000..a122ca8a --- /dev/null +++ b/runtime/strmsrv.c @@ -0,0 +1,966 @@ +/* strmsrv.c + * + * This builds a basic stream server. It handles connection creation but + * not any protocol. Instead, it calls a "data received" entry point of the + * caller with any data received, in which case the caller must react accordingly. + * This module works together with the netstream drivers. + * + * There are actually two classes within the stream server code: one is + * the strmsrv itself, the other one is its sessions. This is a helper + * class to strmsrv. + * + * File begun on 2009-06-01 by RGerhards based on strmsrv.c. Note that strmsrv is + * placed under LGPL, which is possible because I carefully evaluated and + * eliminated all those parts of strmsrv which were not written by me. + * + * TODO: I would consider it useful to migrate tcpsrv.c/tcps_sess.c to this stream + * class here. The requires a little bit redesign, but should not be too hard. The + * core idea, already begun here, is that we still support lots of callbacks, but + * provide "canned" implementations for standard cases. That way, most upper-layer + * modules can be kept rather simple and without any extra overhead. Note that + * to support this, tcps_sess.c would need to extract the message reception state + * machine to a separate module which then is called via the DoCharRcvd() interface + * of this class here. -- rgerhards, 2009-06-01 + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <stdlib.h> +#include <assert.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <stdarg.h> +#include <ctype.h> +#include <netinet/in.h> +#include <netdb.h> +#include <sys/types.h> +#include <sys/socket.h> +#if HAVE_FCNTL_H +#include <fcntl.h> +#endif +#include "rsyslog.h" +#include "dirty.h" +#include "cfsysline.h" +#include "module-template.h" +#include "net.h" +#include "srUtils.h" +#include "conf.h" +#include "strmsrv.h" +#include "obj.h" +#include "glbl.h" +#include "netstrms.h" +#include "netstrm.h" +#include "nssel.h" +#include "errmsg.h" +#include "unicode-helper.h" + +MODULE_TYPE_LIB + +/* defines */ +#define STRMSESS_MAX_DEFAULT 200 /* default for nbr of strm sessions if no number is given */ +#define STRMLSTN_MAX_DEFAULT 20 /* default for nbr of listeners */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(conf) +DEFobjCurrIf(glbl) +DEFobjCurrIf(strms_sess) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(net) +DEFobjCurrIf(netstrms) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(nssel) + +/* forward definitions */ +static rsRetVal create_strm_socket(strmsrv_t *pThis); + +/* standard callbacks, if the caller did not provide us with them (this helps keep us + * flexible while at the same time permits very simple upper-layer modules) + */ +/* this shall go into a specific ACL module! */ +static int +isPermittedHost(struct sockaddr __attribute__((unused)) *addr, char __attribute__((unused)) *fromHostFQDN, + void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) +{ + return 1; +} + + +static rsRetVal +doOpenLstnSocks(strmsrv_t *pSrv) +{ + ISOBJ_TYPE_assert(pSrv, strmsrv); + return create_strm_socket(pSrv); +} + + +static rsRetVal +doRcvData(strms_sess_t *pSess, char *buf, size_t lenBuf, ssize_t *piLenRcvd) +{ + DEFiRet; + assert(pSess != NULL); + assert(piLenRcvd != NULL); + + *piLenRcvd = lenBuf; + CHKiRet(netstrm.Rcv(pSess->pStrm, (uchar*) buf, piLenRcvd)); +finalize_it: + RETiRet; +} + +static rsRetVal +onRegularClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + /* process any incomplete frames left over */ + //strms_sess.PrepareClose(pSess); + /* Session closed */ + strms_sess.Close(pSess); + RETiRet; +} + + +static rsRetVal +onErrClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + strms_sess.Close(pSess); + RETiRet; +} + +/* ------------------------------ end callbacks ------------------------------ */ + +/* add new listener port to listener port list + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +addNewLstnPort(strmsrv_t *pThis, uchar *pszPort) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* create entry */ + CHKmalloc(pEntry = MALLOC(sizeof(strmLstnPortList_t))); + pEntry->pszPort = pszPort; + pEntry->pSrv = pThis; + CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + + /* and add to list */ + pEntry->pNext = pThis->pLstnPorts; + pThis->pLstnPorts = pEntry; + +finalize_it: + RETiRet; +} + + +/* configure STRM listener settings. + * Note: pszPort is handed over to us - the caller MUST NOT free it! + * rgerhards, 2008-03-20 + */ +static rsRetVal +configureSTRMListen(strmsrv_t *pThis, uchar *pszPort) +{ + int i; + uchar *pPort = pszPort; + DEFiRet; + + assert(pszPort != NULL); + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* extract port */ + i = 0; + while(isdigit((int) *pPort)) { + i = i * 10 + *pPort++ - '0'; + } + + if(i >= 0 && i <= 65535) { + CHKiRet(addNewLstnPort(pThis, pszPort)); + } else { + errmsg.LogError(0, NO_ERRCODE, "Invalid STRM listen port %s - ignored.\n", pszPort); + } + +finalize_it: + RETiRet; +} + + +/* Initialize the session table + * returns 0 if OK, somewhat else otherwise + */ +static rsRetVal +STRMSessTblInit(strmsrv_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions == NULL); + + dbgprintf("Allocating buffer for %d STRM sessions.\n", pThis->iSessMax); + if((pThis->pSessions = (strms_sess_t **) calloc(pThis->iSessMax, sizeof(strms_sess_t *))) == NULL) { + dbgprintf("Error: STRMSessInit() could not alloc memory for STRM session table.\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + +finalize_it: + RETiRet; +} + + +/* find a free spot in the session table. If the table + * is full, -1 is returned, else the index of the free + * entry (0 or higher). + */ +static int +STRMSessTblFindFreeSpot(strmsrv_t *pThis) +{ + register int i; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + for(i = 0 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] == NULL) + break; + } + + return((i < pThis->iSessMax) ? i : -1); +} + + +/* Get the next session index. Free session tables entries are + * skipped. This function is provided the index of the last + * session entry, or -1 if no previous entry was obtained. It + * returns the index of the next session or -1, if there is no + * further entry in the table. Please note that the initial call + * might as well return -1, if there is no session at all in the + * session table. + */ +static int +STRMSessGetNxtSess(strmsrv_t *pThis, int iCurr) +{ + register int i; + + BEGINfunc + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions != NULL); + for(i = iCurr + 1 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] != NULL) + break; + } + + ENDfunc + return((i < pThis->iSessMax) ? i : -1); +} + + +/* De-Initialize STRM listner sockets. + * This function deinitializes everything, including freeing the + * session table. No STRM listen receive operations are permitted + * unless the subsystem is reinitialized. + * rgerhards, 2007-06-21 + */ +static void deinit_strm_listener(strmsrv_t *pThis) +{ + int i; + strmLstnPortList_t *pEntry; + strmLstnPortList_t *pDel; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + if(pThis->pSessions != NULL) { + /* close all STRM connections! */ + i = STRMSessGetNxtSess(pThis, -1); + while(i != -1) { + strms_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = STRMSessGetNxtSess(pThis, i); + } + + /* we are done with the session table - so get rid of it... */ + free(pThis->pSessions); + pThis->pSessions = NULL; /* just to make sure... */ + } + + /* free list of strm listen ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + free(pEntry->pszPort); + free(pEntry->pszInputName); + pDel = pEntry; + pEntry = pEntry->pNext; + free(pDel); + } + + /* finally close our listen streams */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + netstrm.Destruct(pThis->ppLstn + i); + } +} + + +/* add a listen socket to our listen socket array. This is a callback + * invoked from the netstrm class. -- rgerhards, 2008-04-23 + */ +static rsRetVal +addStrmLstn(void *pUsr, netstrm_t *pLstn) +{ + strmLstnPortList_t *pPortList = (strmLstnPortList_t *) pUsr; + strmsrv_t *pThis = pPortList->pSrv; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + ISOBJ_TYPE_assert(pLstn, netstrm); + + if(pThis->iLstnMax >= STRMLSTN_MAX_DEFAULT) + ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); + + pThis->ppLstn[pThis->iLstnMax] = pLstn; + pThis->ppLstnPort[pThis->iLstnMax] = pPortList; + ++pThis->iLstnMax; + +finalize_it: + RETiRet; +} + + +/* Initialize STRM listener socket for a single port + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +initSTRMListener(strmsrv_t *pThis, strmLstnPortList_t *pPortEntry) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pPortEntry != NULL); + + /* TODO: add capability to specify local listen address! */ + CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pPortEntry, addStrmLstn, pPortEntry->pszPort, NULL, pThis->iSessMax)); + +finalize_it: + RETiRet; +} + + +/* Initialize STRM sockets (for listener) and listens on them */ +static rsRetVal +create_strm_socket(strmsrv_t *pThis) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* init all configured ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + CHKiRet(initSTRMListener(pThis, pEntry)); + pEntry = pEntry->pNext; + } + + /* OK, we had success. Now it is also time to + * initialize our connections + */ + if(STRMSessTblInit(pThis) != 0) { + /* OK, we are in some trouble - we could not initialize the + * session table, so we can not continue. We need to free all + * we have assigned so far, because we can not really use it... + */ + errmsg.LogError(0, RS_RET_ERR, "Could not initialize STRM session table, suspending STRM message reception."); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Accept new STRM connection; make entry in session table. If there + * is no more space left in the connection table, the new STRM + * connection is immediately dropped. + * ppSess has a pointer to the newly created session, if it succeeds. + * If it does not succeed, no session is created and ppSess is + * undefined. If the user has provided an OnSessAccept Callback, + * this one is executed immediately after creation of the + * session object, so that it can do its own initialization. + * rgerhards, 2008-03-02 + */ +static rsRetVal +SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSess, netstrm_t *pStrm) +{ + DEFiRet; + strms_sess_t *pSess = NULL; + netstrm_t *pNewStrm = NULL; + int iSess = -1; + struct sockaddr_storage *addr; + uchar *fromHostFQDN = NULL; + uchar *fromHostIP = NULL; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pLstnInfo != NULL); + + CHKiRet(netstrm.AcceptConnReq(pStrm, &pNewStrm)); + + /* Add to session list */ + iSess = STRMSessTblFindFreeSpot(pThis); + if(iSess == -1) { + errno = 0; + errmsg.LogError(0, RS_RET_MAX_SESS_REACHED, "too many strm sessions - dropping incoming request"); + ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); + } + + if(pThis->bUseKeepAlive) { + CHKiRet(netstrm.EnableKeepAlive(pNewStrm)); + } + + /* we found a free spot and can construct our session object */ + CHKiRet(strms_sess.Construct(&pSess)); + CHKiRet(strms_sess.SetStrmsrv(pSess, pThis)); + CHKiRet(strms_sess.SetLstnInfo(pSess, pLstnInfo)); + + /* get the host name */ + CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); + CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); + CHKiRet(netstrm.GetRemAddr(pNewStrm, &addr)); + /* TODO: check if we need to strip the domain name here -- rgerhards, 2008-04-24 */ + + /* Here we check if a host is permitted to send us messages. If it isn't, we do not further + * process the message but log a warning (if we are configured to do this). + * rgerhards, 2005-09-26 + */ + if(pThis->pIsPermittedHost != NULL + && !pThis->pIsPermittedHost((struct sockaddr*) addr, (char*) fromHostFQDN, pThis->pUsr, pSess->pUsr)) { + dbgprintf("%s is not an allowed sender\n", fromHostFQDN); + if(glbl.GetOption_DisallowWarning()) { + errno = 0; + errmsg.LogError(0, RS_RET_HOST_NOT_PERMITTED, "STRM message from disallowed sender %s discarded", fromHostFQDN); + } + ABORT_FINALIZE(RS_RET_HOST_NOT_PERMITTED); + } + + /* OK, we have an allowed sender, so let's continue, what + * means we can finally fill in the session object. + */ + CHKiRet(strms_sess.SetHost(pSess, fromHostFQDN)); + fromHostFQDN = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetHostIP(pSess, fromHostIP)); + fromHostIP = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetStrm(pSess, pNewStrm)); + pNewStrm = NULL; /* prevent it from being freed in error handler, now done in strms_sess! */ + CHKiRet(strms_sess.ConstructFinalize(pSess)); + + /* check if we need to call our callback */ + if(pThis->pOnSessAccept != NULL) { + CHKiRet(pThis->pOnSessAccept(pThis, pSess)); + } + + *ppSess = pSess; + pThis->pSessions[iSess] = pSess; + pSess = NULL; /* this is now also handed over */ + +finalize_it: + if(iRet != RS_RET_OK) { + if(pSess != NULL) + strms_sess.Destruct(&pSess); + if(pNewStrm != NULL) + netstrm.Destruct(&pNewStrm); + free(fromHostFQDN); + free(fromHostIP); + } + + RETiRet; +} + + +static void +RunCancelCleanup(void *arg) +{ + nssel_t **ppSel = (nssel_t**) arg; + + if(*ppSel != NULL) + nssel.Destruct(ppSel); +} + + +/* This function is called to gather input. */ +#pragma GCC diagnostic ignored "-Wempty-body" +static rsRetVal +Run(strmsrv_t *pThis) +{ + DEFiRet; + int nfds; + int i; + int iSTRMSess; + int bIsReady; + strms_sess_t *pNewSess; + nssel_t *pSel; + ssize_t iRcvd; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerharsd, 20080-04-24 + */ + pthread_cleanup_push(RunCancelCleanup, (void*) &pSel); + while(1) { + CHKiRet(nssel.Construct(&pSel)); + // TODO: set driver + CHKiRet(nssel.ConstructFinalize(pSel)); + + /* Add the STRM listen sockets to the list of read descriptors. */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.Add(pSel, pThis->ppLstn[i], NSDSEL_RD)); + } + + /* do the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(iSTRMSess != -1) { + /* TODO: access to pNsd is NOT really CLEAN, use method... */ + CHKiRet(nssel.Add(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD)); + /* now get next... */ + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + + /* wait for io to become ready */ + CHKiRet(nssel.Wait(pSel, &nfds)); + + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + --nfds; /* indicate we have processed one */ + } + } + + /* now check the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(nfds && iSTRMSess != -1) { + CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + dbgprintf("netstream %p with new data\n", pThis->pSessions[iSTRMSess]->pStrm); + + /* Receive message */ + iRet = pThis->pRcvData(pThis->pSessions[iSTRMSess], buf, sizeof(buf), &iRcvd); + switch(iRet) { + case RS_RET_CLOSED: + pThis->pOnRegularClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + case RS_RET_RETRY: + /* we simply ignore retry - this is not an error, but we also have not received anything */ + break; + case RS_RET_OK: + /* valid data received, process it! */ + if(strms_sess.DataRcvd(pThis->pSessions[iSTRMSess], buf, iRcvd) != RS_RET_OK) { + /* in this case, something went awfully wrong. + * We are instructed to terminate the session. + */ + errmsg.LogError(0, NO_ERRCODE, "Tearing down STRM Session %d - see " + "previous messages for reason(s)\n", iSTRMSess); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + } + break; + default: + errno = 0; + errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", + pThis->pSessions[iSTRMSess]->pStrm); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + } + --nfds; /* indicate we have processed one */ + } + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + CHKiRet(nssel.Destruct(&pSel)); +finalize_it: /* this is a very special case - this time only we do not exit the function, + * because that would not help us either. So we simply retry it. Let's see + * if that actually is a better idea. Exiting the loop wasn't we always + * crashed, which made sense (the rest of the engine was not prepared for + * that) -- rgerhards, 2008-05-19 + */ + /*EMPTY*/; + } + + /* note that this point is usually not reached */ + pthread_cleanup_pop(0); /* remove cleanup handler */ + + RETiRet; +} +#pragma GCC diagnostic warning "-Wempty-body" + + +/* Standard-Constructor */ +BEGINobjConstruct(strmsrv) /* be sure to specify the object type also in END macro! */ + pThis->iSessMax = STRMSESS_MAX_DEFAULT; /* TODO: useful default ;) */ + /* set default callbacks (used if caller does not overwrite them) */ + pThis->pIsPermittedHost = isPermittedHost; + pThis->OpenLstnSocks = doOpenLstnSocks; + pThis->pRcvData = doRcvData; + pThis->pOnRegularClose = onRegularClose; + pThis->pOnErrClose = onErrClose; + /* session specific callbacks */ + //pThis->OnSessConstructFinalize = + //pThis->pOnSessDestruct = +ENDobjConstruct(strmsrv) + + +/* ConstructionFinalizer */ +static rsRetVal +strmsrvConstructFinalize(strmsrv_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* prepare network stream subsystem */ + CHKiRet(netstrms.Construct(&pThis->pNS)); + CHKiRet(netstrms.SetDrvrMode(pThis->pNS, pThis->iDrvrMode)); + if(pThis->pszDrvrAuthMode != NULL) + CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); + if(pThis->pPermPeers != NULL) + CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); + // TODO: set driver! + CHKiRet(netstrms.ConstructFinalize(pThis->pNS)); + + /* set up listeners */ + CHKmalloc(pThis->ppLstn = calloc(STRMLSTN_MAX_DEFAULT, sizeof(netstrm_t*))); + CHKmalloc(pThis->ppLstnPort = calloc(STRMLSTN_MAX_DEFAULT, sizeof(strmLstnPortList_t*))); + iRet = pThis->OpenLstnSocks(pThis); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + } + RETiRet; +} + + +/* destructor for the strmsrv object */ +BEGINobjDestruct(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strmsrv) + if(pThis->OnDestruct != NULL) + pThis->OnDestruct(pThis->pUsr); + + deinit_strm_listener(pThis); + + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + free(pThis->pszDrvrAuthMode); + free(pThis->ppLstn); + free(pThis->ppLstnPort); + free(pThis->pszInputName); +ENDobjDestruct(strmsrv) + + +/* debugprint for the strmsrv object */ +BEGINobjDebugPrint(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strmsrv) +ENDobjDebugPrint(strmsrv) + +/* set functions */ +static rsRetVal +SetCBIsPermittedHost(strmsrv_t *pThis, int (*pCB)(struct sockaddr *addr, char *fromHostFQDN, void*, void*)) +{ + DEFiRet; + pThis->pIsPermittedHost = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessAccept(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*, strms_sess_t*)) +{ + DEFiRet; + pThis->pOnSessAccept = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessConstructFinalize(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnSessConstructFinalize = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->pOnSessDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnRegularClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnRegularClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnErrClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnErrClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOpenLstnSocks(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*)) +{ + DEFiRet; + pThis->OpenLstnSocks = pCB; + RETiRet; +} + +static rsRetVal +SetUsrP(strmsrv_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + +static rsRetVal +SetKeepAlive(strmsrv_t *pThis, int iVal) +{ + DEFiRet; + dbgprintf("keep-alive set to %d\n", iVal); + pThis->bUseKeepAlive = iVal; + RETiRet; +} + +static rsRetVal +SetOnCharRcvd(strmsrv_t *pThis, rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar)) +{ + DEFiRet; + assert(OnCharRcvd != NULL); + pThis->OnCharRcvd = OnCharRcvd; + RETiRet; +} + +/* Set the input name to use -- rgerhards, 2008-12-10 */ +static rsRetVal +SetInputName(strmsrv_t *pThis, uchar *name) +{ + uchar *pszName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + if(name == NULL) + pszName = NULL; + else + CHKmalloc(pszName = ustrdup(name)); + free(pThis->pszInputName); + pThis->pszInputName = pszName; +finalize_it: + RETiRet; +} + + +/* here follows a number of methods that shuffle authentication settings down + * to the drivers. Drivers not supporting these settings may return an error + * state. + * -------------------------------------------------------------------------- */ + +/* set the driver mode -- rgerhards, 2008-04-30 */ +static rsRetVal +SetDrvrMode(strmsrv_t *pThis, int iMode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iDrvrMode = iMode; + RETiRet; +} + + +/* set the driver authentication mode -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrAuthMode(strmsrv_t *pThis, uchar *mode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + CHKmalloc(pThis->pszDrvrAuthMode = ustrdup(mode)); +finalize_it: + RETiRet; +} + + +/* set the driver's permitted peers -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrPermPeers(strmsrv_t *pThis, permittedPeers_t *pPermPeers) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->pPermPeers = pPermPeers; + RETiRet; +} + + +/* End of methods to shuffle autentication settings to the driver.; + + * -------------------------------------------------------------------------- */ + + +/* set max number of sessions + * this must be called before ConstructFinalize, or it will have no effect! + * rgerhards, 2009-04-09 + */ +static rsRetVal +SetSessMax(strmsrv_t *pThis, int iMax) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iSessMax = iMax; + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strmsrv) +CODESTARTobjQueryInterface(strmsrv) + if(pIf->ifVersion != strmsrvCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strmsrvDebugPrint; + pIf->Construct = strmsrvConstruct; + pIf->ConstructFinalize = strmsrvConstructFinalize; + pIf->Destruct = strmsrvDestruct; + + pIf->configureSTRMListen = configureSTRMListen; + pIf->create_strm_socket = create_strm_socket; + pIf->Run = Run; + + pIf->SetKeepAlive = SetKeepAlive; + pIf->SetUsrP = SetUsrP; + pIf->SetInputName = SetInputName; + pIf->SetSessMax = SetSessMax; + pIf->SetDrvrMode = SetDrvrMode; + pIf->SetDrvrAuthMode = SetDrvrAuthMode; + pIf->SetDrvrPermPeers = SetDrvrPermPeers; + pIf->SetCBIsPermittedHost = SetCBIsPermittedHost; + pIf->SetCBOpenLstnSocks = SetCBOpenLstnSocks; + pIf->SetCBOnSessAccept = SetCBOnSessAccept; + pIf->SetCBOnSessConstructFinalize = SetCBOnSessConstructFinalize; + pIf->SetCBOnSessDestruct = SetCBOnSessDestruct; + pIf->SetCBOnDestruct = SetCBOnDestruct; + pIf->SetCBOnRegularClose = SetCBOnRegularClose; + pIf->SetCBOnErrClose = SetCBOnErrClose; + pIf->SetOnCharRcvd = SetOnCharRcvd; + +finalize_it: +ENDobjQueryInterface(strmsrv) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strmsrv, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strmsrv) + /* release objects we no longer need */ + objRelease(strms_sess, DONT_LOAD_LIB); + objRelease(conf, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrms, DONT_LOAD_LIB); + objRelease(nssel, DONT_LOAD_LIB); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(net, LM_NET_FILENAME); +ENDObjClassExit(strmsrv) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strmsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(net, LM_NET_FILENAME)); + CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); + CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(strms_sess, DONT_LOAD_LIB)); + CHKiRet(objUse(conf, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strmsrvDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmsrvConstructFinalize); +ENDObjClassInit(strmsrv) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit + /* de-init in reverse order! */ + strmsrvClassExit(); + strms_sessClassExit(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + /* Initialize all classes that are in our module - this includes ourselfs */ + CHKiRet(strms_sessClassInit(pModInfo)); + CHKiRet(strmsrvClassInit(pModInfo)); /* must be done after strms_sess, as we use it */ +ENDmodInit + +/* vim:set ai: + */ diff --git a/runtime/strmsrv.h b/runtime/strmsrv.h new file mode 100644 index 00000000..86e529c3 --- /dev/null +++ b/runtime/strmsrv.h @@ -0,0 +1,112 @@ +/* Definitions for strmsrv class. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMSRV_H +#define INCLUDED_STRMSRV_H + +#include "obj.h" +#include "strms_sess.h" + +/* list of strm listen ports */ +struct strmLstnPortList_s { + uchar *pszPort; /**< the ports the listener shall listen on */ + uchar *pszInputName; /**< value to be used as input name */ + strmsrv_t *pSrv; /**< pointer to higher-level server instance */ + strmLstnPortList_t *pNext; /**< next port or NULL */ +}; + + +/* the strmsrv object */ +struct strmsrv_s { + BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ + int bUseKeepAlive; /**< use socket layer KEEPALIVE handling? */ + netstrms_t *pNS; /**< pointer to network stream subsystem */ + int iDrvrMode; /**< mode of the stream driver to use */ + uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ + uchar *pszInputName; /**< value to be used as input name */ + permittedPeers_t *pPermPeers;/**< driver's permitted peers */ + int iLstnMax; /**< max nbr of listeners currently supported */ + netstrm_t **ppLstn; /**< our netstream listners */ + strmLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ + int iSessMax; /**< max number of sessions supported */ + strmLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ + int addtlFrameDelim; /**< additional frame delimiter for plain STRM syslog framing (e.g. to handle NetScreen) */ + strms_sess_t **pSessions;/**< array of all of our sessions */ + void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ + /* callbacks */ + int (*pIsPermittedHost)(struct sockaddr *addr, char *fromHostFQDN, void*pUsrSrv, void*pUsrSess); + rsRetVal (*pRcvData)(strms_sess_t*, char*, size_t, ssize_t *); + rsRetVal (*OpenLstnSocks)(struct strmsrv_s*); + rsRetVal (*pOnListenDeinit)(void*); + rsRetVal (*OnDestruct)(void*); + rsRetVal (*pOnRegularClose)(strms_sess_t *pSess); + rsRetVal (*pOnErrClose)(strms_sess_t *pSess); + /* session specific callbacks */ + rsRetVal (*pOnSessAccept)(strmsrv_t *, strms_sess_t*); + rsRetVal (*OnSessConstructFinalize)(void*); + rsRetVal (*pOnSessDestruct)(void*); + rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar); +}; + + +/* interfaces */ +BEGINinterface(strmsrv) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strmsrv); + rsRetVal (*Construct)(strmsrv_t **ppThis); + rsRetVal (*ConstructFinalize)(strmsrv_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strmsrv_t **ppThis); + rsRetVal (*configureSTRMListen)(strmsrv_t*, uchar *pszPort); + //rsRetVal (*SessAccept)(strmsrv_t *pThis, strmLstnPortList_t*, strms_sess_t **ppSess, netstrm_t *pStrm); + rsRetVal (*create_strm_socket)(strmsrv_t *pThis); + rsRetVal (*Run)(strmsrv_t *pThis); + /* set methods */ + rsRetVal (*SetAddtlFrameDelim)(strmsrv_t*, int); + rsRetVal (*SetInputName)(strmsrv_t*, uchar*); + rsRetVal (*SetKeepAlive)(strmsrv_t*, int); + rsRetVal (*SetUsrP)(strmsrv_t*, void*); + rsRetVal (*SetCBIsPermittedHost)(strmsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); + rsRetVal (*SetCBOpenLstnSocks)(strmsrv_t *, rsRetVal (*)(strmsrv_t*)); + rsRetVal (*SetCBOnDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnRegularClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetCBOnErrClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetDrvrMode)(strmsrv_t *pThis, int iMode); + rsRetVal (*SetDrvrAuthMode)(strmsrv_t *pThis, uchar *pszMode); + rsRetVal (*SetDrvrPermPeers)(strmsrv_t *pThis, permittedPeers_t*); + /* session specifics */ + rsRetVal (*SetCBOnSessAccept)(strmsrv_t*, rsRetVal (*) (strmsrv_t*, strms_sess_t*)); + rsRetVal (*SetCBOnSessDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnSessConstructFinalize)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetSessMax)(strmsrv_t *pThis, int iMaxSess); + rsRetVal (*SetOnCharRcvd)(strmsrv_t *pThis, rsRetVal (*OnMsgCharRcvd)(strms_sess_t*, uchar)); +ENDinterface(strmsrv) +#define strmsrvCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* change for v?: + */ + + +/* prototypes */ +PROTOTYPEObj(strmsrv); + +/* the name of our library binary */ +#define LM_STRMSRV_FILENAME "lmstrmsrv" + +#endif /* #ifndef INCLUDED_STRMSRV_H */ diff --git a/runtime/sync.c b/runtime/sync.c index a3053e28..15d5c80f 100644 --- a/runtime/sync.c +++ b/runtime/sync.c @@ -28,12 +28,13 @@ #include "rsyslog.h" #include "sync.h" +#include "debug.h" void SyncObjInit(pthread_mutex_t **mut) { - *mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + *mut = (pthread_mutex_t *) MALLOC(sizeof (pthread_mutex_t)); pthread_mutex_init(*mut, NULL); } diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h index be0dfdd8..fcebd985 100644 --- a/runtime/syslogd-types.h +++ b/runtime/syslogd-types.h @@ -56,7 +56,10 @@ * applications I do not yet envision. -- rgerhards, 2007-07-24 */ typedef enum _syslogFeature { - sFEATURERepeatedMsgReduction = 1 + sFEATURERepeatedMsgReduction = 1, /* for output modules */ + sFEATURENonCancelInputTermination = 2, /* for input modules */ + sFEATUREAutomaticSanitazion = 3, /* for parser modules */ + sFEATUREAutomaticPRIParsing = 4 /* for parser modules */ } syslogFeature; /* we define our own facility and severities */ @@ -76,25 +79,31 @@ enum _EHostnameCmpMode { }; typedef enum _EHostnameCmpMode EHostnameCmpMode; +/* time type numerical values for structure below */ +#define TIME_TYPE_UNINIT 0 +#define TIME_TYPE_RFC3164 1 +#define TIME_TYPE_RFC5424 2 /* rgerhards 2004-11-11: the following structure represents * a time as it is used in syslog. + * rgerhards, 2009-06-23: packed structure for better cache performance + * (but left ultimate decision about packing to compiler) */ struct syslogTime { - int timeType; /* 0 - unitinialized , 1 - RFC 3164, 2 - syslog-protocol */ - int year; - int month; - int day; - int hour; /* 24 hour clock */ - int minute; - int second; - int secfrac; /* fractional seconds (must be 32 bit!) */ - int secfracPrecision; + intTiny timeType; /* 0 - unitinialized , 1 - RFC 3164, 2 - syslog-protocol */ + intTiny month; + intTiny day; + intTiny hour; /* 24 hour clock */ + intTiny minute; + intTiny second; + intTiny secfracPrecision; + intTiny OffsetMinute; /* UTC offset in minutes */ + intTiny OffsetHour; /* UTC offset in hours + * full UTC offset minutes = OffsetHours*60 + OffsetMinute. Then use + * OffsetMode to know the direction. + */ char OffsetMode; /* UTC offset + or - */ - char OffsetHour; /* UTC offset in hours */ - int OffsetMinute; /* UTC offset in minutes */ - /* full UTC offset minutes = OffsetHours*60 + OffsetMinute. Then use - * OffsetMode to know the direction. - */ + short year; + int secfrac; /* fractional seconds (must be 32 bit!) */ }; typedef struct syslogTime syslogTime_t; diff --git a/runtime/sysvar.c b/runtime/sysvar.c index c102d1f5..4a6ace19 100644 --- a/runtime/sysvar.c +++ b/runtime/sysvar.c @@ -175,8 +175,6 @@ CODESTARTobjQueryInterface(sysvar) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - //xxxpIf->oID = "sysvar";//OBJsysvar; - pIf->Construct = sysvarConstruct; pIf->ConstructFinalize = sysvarConstructFinalize; pIf->Destruct = sysvarDestruct; diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 36d76a78..7a776f68 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -4,6 +4,9 @@ * The following functions are wrappers which hopefully enable us to move * from 8-bit chars to unicode with relative ease when we finally attack this * + * Note: while we prefer inline functions, this leads to invalid references in + * core dumps. So in a debug build, we use macros where appropriate... + * * Begun 2009-05-21 RGerhards * * Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH @@ -31,6 +34,22 @@ #include <string.h> +#ifdef DEBUG +# define ustrncpy(psz1, psz2, len) strncpy((char*)(psz1), (char*)(psz2), (len)) +# define ustrdup(psz) (uchar*)strdup((char*)(psz)) +#else + static inline uchar* ustrncpy(uchar *psz1, uchar *psz2, size_t len) + { + return (uchar*) strncpy((char*) psz1, (char*) psz2, len); + } + + static inline uchar* ustrdup(uchar *psz) + { + return (uchar*) strdup((char*)psz); + } + +#endif /* #ifdef DEBUG */ + static inline int ustrcmp(uchar *psz1, uchar *psz2) { return strcmp((char*) psz1, (char*) psz2); @@ -41,13 +60,9 @@ static inline int ustrlen(uchar *psz) return strlen((char*) psz); } -static inline uchar* ustrdup(uchar *psz) -{ - return (uchar*) strdup((char*)psz); -} - #define UCHAR_CONSTANT(x) ((uchar*) (x)) +#define CHAR_CONVERT(x) ((char*) (x)) #endif /* multi-include protection */ /* vim:set ai: diff --git a/runtime/unlimited_select.h b/runtime/unlimited_select.h new file mode 100644 index 00000000..32dadc03 --- /dev/null +++ b/runtime/unlimited_select.h @@ -0,0 +1,45 @@ +/* unlimited_select.h + * Tweak the macros for accessing fd_set so that the select() syscall + * won't be limited to a particular number of file descriptors. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +#ifndef UNLIMITED_SELECT_H_INCLUDED + +#include <string.h> +#include <stdlib.h> +#include <sys/select.h> +#include "glbl.h" + +#ifdef USE_UNLIMITED_SELECT +# undef FD_ZERO +# define FD_ZERO(set) memset((set), 0, glbl.GetFdSetSize()); +#endif + +#ifdef USE_UNLIMITED_SELECT +void freeFdSet(fd_set *p) { + free(p); +} +#else +# define freeFdSet(x) +#endif + +#endif /* #ifndef UNLIMITED_SELECT_H_INCLUDED */ diff --git a/runtime/vm.c b/runtime/vm.c index 125b0d21..0ed174d1 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -34,6 +34,7 @@ #include "vm.h" #include "sysvar.h" #include "stringbuf.h" +#include "unicode-helper.h" /* static data */ DEFobjStaticHelpers @@ -41,6 +42,8 @@ DEFobjCurrIf(vmstk) DEFobjCurrIf(var) DEFobjCurrIf(sysvar) +static pthread_mutex_t mutGetenv; /* we need to make this global because otherwise we can not guarantee proper init! */ + /* ------------------------------ function registry code and structures ------------------------------ */ /* we maintain a registry of known functions */ @@ -82,11 +85,15 @@ rsfrAddFunction(uchar *szName, prsf_t rsf) /* unique name, so add to head of list */ CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + CHKiRet(cstrFinalize(pEntry->pName)); pEntry->rsf = rsf; pEntry->pNext = funcRegRoot; funcRegRoot = pEntry; finalize_it: + if(iRet != RS_RET_OK && iRet != RS_RET_DUP_FUNC_NAME) + free(pEntry); + RETiRet; } @@ -167,7 +174,7 @@ rsfrRemoveAll(void) while(pEntry != NULL) { pEntryDel = pEntry; pEntry = pEntry->pNext; - rsCStrDestruct(&pEntryDel->pName); + cstrDestruct(&pEntryDel->pName); free(pEntryDel); } funcRegRoot = NULL; @@ -405,6 +412,7 @@ CODESTARTop(STRADD) vmstk.PopString(pThis->pStk, &operand1); CHKiRet(rsCStrAppendCStr(operand1->val.pStr, operand2->val.pStr)); + CHKiRet(cstrFinalize(operand1->val.pStr)); /* we have a result, so let's push it */ vmstk.Push(pThis->pStk, operand1); @@ -537,6 +545,42 @@ finalize_it: } +/* The getenv function. Note that we guard the OS call by a mutex, as that + * function is not guaranteed to be thread-safe. This implementation here is far from + * being optimal, at least we should cache the result. This is left TODO for + * a later revision. + * rgerhards, 2009-11-03 + */ +static rsRetVal +rsf_getenv(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + char *envResult; + cstr_t *pCstr; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton (trivial case here...) */ + vmstk.PopString(pStk, &operand1); + d_pthread_mutex_lock(&mutGetenv); + envResult = getenv((char*) rsCStrGetSzStr(operand1->val.pStr)); + DBGPRINTF("rsf_getenv(): envvar '%s', return '%s'\n", rsCStrGetSzStr(operand1->val.pStr), + envResult == NULL ? "(NULL)" : envResult); + iRet = rsCStrConstructFromszStr(&pCstr, (envResult == NULL) ? UCHAR_CONSTANT("") : (uchar*)envResult); + d_pthread_mutex_unlock(&mutGetenv); + if(iRet != RS_RET_OK) + FINALIZE; /* need to do this after mutex is unlocked! */ + + /* Store result and cleanup */ + var.SetString(operand1, pCstr); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + /* The "tolower" function, which converts its sole argument to lower case. * Quite honestly, currently this is primarily a test driver for me... * rgerhards, 2009-04-06 @@ -554,16 +598,16 @@ rsf_tolower(vmstk_t *pStk, int numOperands) ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); /* pop args and do operaton */ - CHKiRet(rsCStrConstruct(&pcstr)); + CHKiRet(cstrConstruct(&pcstr)); vmstk.PopString(pStk, &operand1); - pSrc = rsCStrGetSzStr(operand1->val.pStr); - iStrlen = strlen((char*)pSrc); + pSrc = cstrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); // TODO: use count from string! while(iStrlen--) { - CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + CHKiRet(cstrAppendChar(pcstr, tolower(*pSrc++))); } /* Store result and cleanup */ - CHKiRet(rsCStrFinish(pcstr)); + CHKiRet(cstrFinalize(pcstr)); var.SetString(operand1, pcstr); vmstk.Push(pStk, operand1); finalize_it: @@ -754,6 +798,8 @@ BEGINObjClassExit(vm, OBJ_IS_CORE_MODULE) /* class, version */ objRelease(sysvar, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(vmstk, CORE_COMPONENT); + + pthread_mutex_destroy(&mutGetenv); ENDObjClassExit(vm) @@ -774,6 +820,9 @@ BEGINObjClassInit(vm, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* register built-in functions // TODO: move to its own module */ CHKiRet(rsfrAddFunction((uchar*)"strlen", rsf_strlen)); CHKiRet(rsfrAddFunction((uchar*)"tolower", rsf_tolower)); + CHKiRet(rsfrAddFunction((uchar*)"getenv", rsf_getenv)); + + pthread_mutex_init(&mutGetenv, NULL); ENDObjClassInit(vm) diff --git a/runtime/vmop.c b/runtime/vmop.c index 3e001d27..ea627220 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -85,7 +85,7 @@ CODESTARTobjDebugPrint(vmop) CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); } } - CHKiRet(rsCStrFinish(&pStrVar)); + CHKiRet(cstrFinalize(pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); if(pThis->opcode != opcode_FUNC_CALL) rsCStrDestruct(&pStrVar); @@ -125,7 +125,7 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) if(pThis->operand.pVar != NULL) CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); } - CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); + CHKiRet(cstrAppendChar(pstrPrg, '\n')); finalize_it: RETiRet; diff --git a/runtime/wti.c b/runtime/wti.c index 544bffa7..14964fb0 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -39,10 +39,10 @@ #include <pthread.h> #include <errno.h> -#ifdef OS_SOLARIS -# include <sched.h> -# define pthread_yield() sched_yield() -#endif +/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 +//#ifdef OS_SOLARIS +//# include <sched.h> +//#endif #include "rsyslog.h" #include "stringbuf.h" @@ -51,6 +51,7 @@ #include "wti.h" #include "obj.h" #include "glbl.h" +#include "atomic.h" /* static data */ DEFobjStaticHelpers @@ -75,85 +76,50 @@ wtiGetDbgHdr(wti_t *pThis) } -/* get the current worker state. For simplicity and speed, we have - * NOT used our regular calling interface this time. I hope that won't - * bite in the long term... -- rgerhards, 2008-01-17 - * TODO: may be performance optimized by atomic operations +/* return the current worker processing state. For the sake of + * simplicity, we do not use the iRet interface. -- rgerhards, 2009-07-17 */ -qWrkCmd_t -wtiGetState(wti_t *pThis, int bLockMutex) +sbool +wtiGetState(wti_t *pThis) { - DEFVARS_mutexProtection; - qWrkCmd_t tCmd; - - BEGINfunc - ISOBJ_TYPE_assert(pThis, wti); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - tCmd = pThis->tCurrCmd; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - ENDfunc - return tCmd; + return ATOMIC_FETCH_32BIT(pThis->bIsRunning); } -/* send a command to a specific thread - * bActiveOnly specifies if the command should be sent only when the worker is - * in an active state. -- rgerhards, 2008-01-20 +/* Set this thread to "always running" state (can not be unset) + * rgerhards, 2009-07-20 */ rsRetVal -wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) +wtiSetAlwaysRunning(wti_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; - ISOBJ_TYPE_assert(pThis, wti); - assert(tCmd <= eWRKTHRD_SHUTDOWN_IMMEDIATE); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - - /* all worker states must be followed sequentially, only termination can be set in any state */ - if( (bActiveOnly && (pThis->tCurrCmd < eWRKTHRD_RUN_CREATED)) - || (pThis->tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) { - dbgprintf("%s: command %d can not be accepted in current %d processing state - ignored\n", - wtiGetDbgHdr(pThis), tCmd, pThis->tCurrCmd); - } else { - dbgprintf("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd); - /* we could replace this with a simple if, but we leave the switch in in case we need - * to add something at a later stage. -- rgerhards, 2008-09-30 - */ - switch(tCmd) { - case eWRKTHRD_TERMINATING: - /* TODO: re-enable meaningful debug msg! (via function callback?) - dbgprintf("%s: thread terminating with %d entries left in queue, %d workers running.\n", - wtiGetDbgHdr(pThis->pQueue), pThis->pQueue->iQueueSize, - pThis->pQueue->iCurNumWrkThrd); - */ - pthread_cond_signal(&pThis->condExitDone); - dbgprintf("%s: worker terminating\n", wtiGetDbgHdr(pThis)); - break; - /* these cases just to satisfy the compiler, we do (yet) not act an them: */ - case eWRKTHRD_RUNNING: - case eWRKTHRD_STOPPED: - case eWRKTHRD_RUN_CREATED: - case eWRKTHRD_RUN_INIT: - case eWRKTHRD_SHUTDOWN: - case eWRKTHRD_SHUTDOWN_IMMEDIATE: - /* DO NOTHING */ - break; - } - pThis->tCurrCmd = tCmd; /* apply the new state */ - } + pThis->bAlwaysRunning = TRUE; + return RS_RET_OK; +} - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - RETiRet; +/* Set status (thread is running or not), actually an property of + * use for wtp, but we need to have it per thread instance (thus it + * is inside wti). -- rgerhards, 2009-07-17 + */ +rsRetVal +wtiSetState(wti_t *pThis, sbool bNewVal) +{ + ISOBJ_TYPE_assert(pThis, wti); + if(bNewVal) + ATOMIC_STORE_1_TO_INT(pThis->bIsRunning); + else + ATOMIC_STORE_0_TO_INT(pThis->bIsRunning); + return RS_RET_OK; } -/* Cancel the thread. If the thread is already cancelled or termination, - * we do not again cancel it. But it is save and legal to call wtiCancelThrd() in - * such situations. +/* Cancel the thread. If the thread is not running. But it is save and legal to + * call wtiCancelThrd() in such situations. This function only returns when the + * thread has terminated. Else we may get race conditions all over the code... + * Note that when waiting for the thread to terminate, we do a busy wait, checking + * progress every 10ms. It is very unlikely that we will ever cancel a thread + * and, if so, it will only happen at the end of the rsyslog run. So doing this + * kind of non-optimal wait is considered preferable over using condition variables. * rgerhards, 2008-02-26 */ rsRetVal @@ -163,17 +129,15 @@ wtiCancelThrd(wti_t *pThis) ISOBJ_TYPE_assert(pThis, wti); - d_pthread_mutex_lock(&pThis->mut); - - if(pThis->tCurrCmd >= eWRKTHRD_TERMINATING) { + if(wtiGetState(pThis)) { dbgoprint((obj_t*) pThis, "canceling worker thread\n"); pthread_cancel(pThis->thrdID); - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pThis->pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + /* now wait until the thread terminates... */ + while(wtiGetState(pThis)) { + srSleep(0, 10000); + } } - d_pthread_mutex_unlock(&pThis->mut); - RETiRet; } @@ -181,37 +145,15 @@ wtiCancelThrd(wti_t *pThis) /* Destructor */ BEGINobjDestruct(wti) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(wti) - /* if we reach this point, we must make sure the associated worker has terminated. It is - * the callers duty to make sure the worker already knows it shall terminate. - * TODO: is it *really* the caller's duty? ...mmmhhhh.... smells bad... rgerhards, 2008-01-25 - */ - wtiProcessThrdChanges(pThis, LOCK_MUTEX); /* process state change one last time */ - - d_pthread_mutex_lock(&pThis->mut); - if(wtiGetState(pThis, MUTEX_ALREADY_LOCKED) != eWRKTHRD_STOPPED) { - dbgprintf("%s: WARNING: worker %p shall be destructed but is still running (might be OK) - joining it\n", - wtiGetDbgHdr(pThis), pThis); - /* let's hope the caller actually instructed it to shutdown... */ - pthread_cond_wait(&pThis->condExitDone, &pThis->mut); - wtiJoinThrd(pThis); - } - d_pthread_mutex_unlock(&pThis->mut); - /* actual destruction */ - pthread_cond_destroy(&pThis->condExitDone); - pthread_mutex_destroy(&pThis->mut); - - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->batch.pElem); + free(pThis->pszDbgHdr); ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); - pthread_cond_init(&pThis->condExitDone, NULL); - pthread_mutex_init(&pThis->mut, NULL); ENDobjConstruct(wti) @@ -222,81 +164,28 @@ rsRetVal wtiConstructFinalize(wti_t *pThis) { DEFiRet; + int iDeqBatchSize; ISOBJ_TYPE_assert(pThis, wti); dbgprintf("%s: finalizing construction of worker instance data\n", wtiGetDbgHdr(pThis)); - /* initialize our thread instance descriptor */ - pThis->pUsrp = NULL; - pThis->tCurrCmd = eWRKTHRD_STOPPED; - - RETiRet; -} - - -/* join a specific worker thread - * we do not lock the mutex, because join will sync anyways... - */ -rsRetVal -wtiJoinThrd(wti_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, wti); - dbgprintf("waiting for worker %s termination, current state %d\n", wtiGetDbgHdr(pThis), pThis->tCurrCmd); - if (pThis->thrdID == 0) { - dbgprintf("worker %s was already stopped\n", wtiGetDbgHdr(pThis)); - } else { - pthread_join(pThis->thrdID, NULL); - wtiSetState(pThis, eWRKTHRD_STOPPED, 0, MUTEX_ALREADY_LOCKED); /* back to virgin... */ - pThis->thrdID = 0; /* invalidate the thread ID so that we do not accidently find reused ones */ - dbgprintf("worker %s has stopped\n", wtiGetDbgHdr(pThis)); - } - - RETiRet; -} - -/* check if we had a worker thread changes and, if so, act - * on it. At a minimum, terminated threads are harvested (joined). - */ -rsRetVal -wtiProcessThrdChanges(wti_t *pThis, int bLockMutex) -{ - DEFiRet; - DEFVARS_mutexProtection; + /* initialize our thread instance descriptor (no concurrency here) */ + pThis->bIsRunning = FALSE; - ISOBJ_TYPE_assert(pThis, wti); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - switch(pThis->tCurrCmd) { - case eWRKTHRD_TERMINATING: - /* we need to at least temporarily release the mutex, because otherwise - * we may deadlock with the thread we intend to join (it aquires the mutex - * during termination processing). -- rgerhards, 2008-02-26 - */ - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - iRet = wtiJoinThrd(pThis); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - break; - /* these cases just to satisfy the compiler, we do not act an them: */ - case eWRKTHRD_STOPPED: - case eWRKTHRD_RUN_CREATED: - case eWRKTHRD_RUN_INIT: - case eWRKTHRD_RUNNING: - case eWRKTHRD_SHUTDOWN: - case eWRKTHRD_SHUTDOWN_IMMEDIATE: - /* DO NOTHING */ - break; - } - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ + CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize)); + CHKmalloc(pThis->batch.pElem = calloc((size_t)iDeqBatchSize, sizeof(batch_obj_t))); +finalize_it: RETiRet; } /* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. + * Most importantly, it must bring back the batch into a consistent state. + * Keep in mind that cancellation is disabled if we run into + * the cancel cleanup handler (and have been cancelled). * rgerhards, 2008-01-16 */ static void @@ -304,7 +193,6 @@ wtiWorkerCancelCleanup(void *arg) { wti_t *pThis = (wti_t*) arg; wtp_t *pWtp; - int iCancelStateSave; BEGINfunc ISOBJ_TYPE_assert(pThis, wti); @@ -312,129 +200,108 @@ wtiWorkerCancelCleanup(void *arg) ISOBJ_TYPE_assert(pWtp, wtp); DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); + pWtp->pfObjProcessed(pWtp->pUsr, pThis); + DBGPRINTF("%s: done cancelation cleanup handler.\n", wtiGetDbgHdr(pThis)); - /* call user supplied handler (that one e.g. requeues the element) */ - pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp); + ENDfunc +} - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pWtp->mut); - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - /* TODO: sync access? I currently think it is NOT needed -- rgerhards, 2008-01-28 */ - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ - d_pthread_mutex_unlock(&pWtp->mut); - pthread_setcancelstate(iCancelStateSave, NULL); +/* wait for queue to become non-empty or timeout + * helper to wtiWorker. Note the the predicate is + * re-tested by the caller, so it is OK to NOT do it here. + * rgerhards, 2009-05-20 + */ +static inline void +doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) +{ + struct timespec t; + + BEGINfunc + DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); + + if(pThis->bAlwaysRunning) { + /* never shut down any started worker */ + d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); + } else { + timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ + if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { + DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); + *pbInactivityTOOccured = 1; /* indicate we had a timeout */ + } + } + dbgoprint((obj_t*) pThis, "worker awoke from idle processing\n"); ENDfunc } -/* generic worker thread framework - * - * Some special comments below, so that they do not clutter the main function code: - * - * On the use of pthread_testcancel(): - * Now make sure we can get canceled - it is not specified if pthread_setcancelstate() is - * a cancellation point in itself. As we run most of the time without cancel enabled, I fear - * we may never get cancelled if we do not create a cancellation point ourselfs. - * - * On the use of pthread_yield(): - * We yield to give the other threads a chance to obtain the mutex. If we do not - * do that, this thread may very well aquire the mutex again before another thread - * has even a chance to run. The reason is that mutex operations are free to be - * implemented in the quickest possible way (and they typically are!). That is, the - * mutex lock/unlock most probably just does an atomic memory swap and does not necessarily - * schedule other threads waiting on the same mutex. That can lead to the same thread - * aquiring the mutex ever and ever again while all others are starving for it. We - * have exactly seen this behaviour when we deliberately introduced a long-running - * test action which basically did a sleep. I understand that with real actions the - * likelihood of this starvation condition is very low - but it could still happen - * and would be very hard to debug. The yield() is a sure fix, its performance overhead - * should be well accepted given the above facts. -- rgerhards, 2008-01-10 +/* generic worker thread framework. Note that we prohibit cancellation + * during almost all times, because it can have very undesired side effects. + * However, we may need to cancel a thread if the consumer blocks for too + * long (during shutdown). So what we do is block cancellation, and every + * consumer must enable it during the periods where it is safe. */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal wtiWorker(wti_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; - struct timespec t; wtp_t *pWtp; /* our worker thread pool */ int bInactivityTOOccured = 0; + rsRetVal localRet; + rsRetVal terminateRet; + int iCancelStateSave; + DEFiRet; ISOBJ_TYPE_assert(pThis, wti); pWtp = pThis->pWtp; /* shortcut */ ISOBJ_TYPE_assert(pWtp, wtp); dbgSetThrdName(pThis->pszDbgHdr); - pThis->pUsrp = NULL; pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - pWtp->pfOnWorkerStartup(pWtp->pUsr); - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); /* now we have our identity, on to real processing */ while(1) { /* loop will be broken below - need to do mutex locks */ - /* process any pending thread requests */ - wtpProcessThrdChanges(pWtp); - pthread_testcancel(); /* see big comment in function header */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); /* see big comment in function header */ -# endif - - /* if we have a rate-limiter set for this worker pool, let's call it. Please - * keep in mind that the rate-limiter may hold us for an extended period - * of time. -- rgerhards, 2008-04-02 - */ - if(pWtp->pfRateLimiter != NULL) { + if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */ pWtp->pfRateLimiter(pWtp->pUsr); } - wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - - if( (bInactivityTOOccured && pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) - || wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED)) { - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); - break; /* end worker thread run */ + d_pthread_mutex_lock(pWtp->pmutUsr); + + /* first check if we are in shutdown process (but evaluate a bit later) */ + terminateRet = wtpChkStopWrkr(pWtp, MUTEX_ALREADY_LOCKED); + if(terminateRet == RS_RET_TERMINATE_NOW) { + /* we now need to free the old batch */ + localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis); + dbgoprint((obj_t*) pThis, "terminating worker because of TERMINATE_NOW mode, del iRet %d\n", + localRet); + d_pthread_mutex_unlock(pWtp->pmutUsr); + break; } - bInactivityTOOccured = 0; /* reset for next run */ - /* if we reach this point, we are still protected by the mutex */ - - if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) { - DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); - pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); - - if(pWtp->toWrkShutdown == -1) { - /* never shut down any started worker */ - d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); - } else { - timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ - if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { - DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); - bInactivityTOOccured = 1; /* indicate we had a timeout */ - } + /* try to execute and process whatever we have */ + /* Note that this function releases and re-aquires the mutex. The returned + * information on idle state must be processed before releasing the mutex again. + */ + localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); + + if(localRet == RS_RET_IDLE) { + if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { + d_pthread_mutex_unlock(pWtp->pmutUsr); + break; /* end of loop */ } - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); + d_pthread_mutex_unlock(pWtp->pmutUsr); continue; /* request next iteration */ } - /* if we reach this point, we have a non-empty queue (and are still protected by mutex) */ - pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); + d_pthread_mutex_unlock(pWtp->pmutUsr); + + bInactivityTOOccured = 0; /* reset for next run */ } /* indicate termination */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pThis->mut); pthread_cleanup_pop(0); /* remove cleanup handler */ - - pWtp->pfOnWorkerShutdown(pWtp->pUsr); - - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ - d_pthread_mutex_unlock(&pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); RETiRet; @@ -463,10 +330,9 @@ wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg) if(pThis->pszDbgHdr != NULL) { free(pThis->pszDbgHdr); - pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ @@ -497,6 +363,5 @@ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wti) -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/wti.h b/runtime/wti.h index 6b60b833..e587c69e 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -1,6 +1,6 @@ /* Definition of the worker thread instance (wti) class. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 by Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -27,22 +27,19 @@ #include <pthread.h> #include "wtp.h" #include "obj.h" +#include "batch.h" + /* the worker thread instance class */ -typedef struct wti_s { +struct wti_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ - pthread_t thrdID; /* thread ID */ - qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ + pthread_t thrdID; /* thread ID */ + int bIsRunning; /* is this thread currently running? (must be int for atomic op!) */ + sbool bAlwaysRunning; /* should this thread always run? */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ - pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */ - pthread_mutex_t mut; - int bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ + batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ uchar *pszDbgHdr; /* header string for debug messages */ -} wti_t; - -/* some symbolic constants for easier reference */ +}; /* prototypes */ @@ -50,12 +47,11 @@ rsRetVal wtiConstruct(wti_t **ppThis); rsRetVal wtiConstructFinalize(wti_t *pThis); rsRetVal wtiDestruct(wti_t **ppThis); rsRetVal wtiWorker(wti_t *pThis); -rsRetVal wtiProcessThrdChanges(wti_t *pThis, int bLockMutex); rsRetVal wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg); -rsRetVal wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex); -rsRetVal wtiJoinThrd(wti_t *pThis); rsRetVal wtiCancelThrd(wti_t *pThis); -qWrkCmd_t wtiGetState(wti_t *pThis, int bLockMutex); +rsRetVal wtiSetAlwaysRunning(wti_t *pThis); +rsRetVal wtiSetState(wti_t *pThis, sbool bNew); +sbool wtiGetState(wti_t *pThis); PROTOTYPEObjClassInit(wti); PROTOTYPEpropSetMeth(wti, pszDbgHdr, uchar*); PROTOTYPEpropSetMeth(wti, pWtp, wtp_t*); diff --git a/runtime/wtp.c b/runtime/wtp.c index 04eb974f..ab7ca4bb 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -8,7 +8,7 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -39,18 +39,23 @@ #include <fcntl.h> #include <unistd.h> #include <errno.h> - -#ifdef OS_SOLARIS -# include <sched.h> -# define pthread_yield() sched_yield() +#include <atomic.h> +#if HAVE_SYS_PRCTL_H +# include <sys/prctl.h> #endif +/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 +//#ifdef OS_SOLARIS +//# include <sched.h> +//#endif + #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" #include "wtp.h" #include "wti.h" #include "obj.h" +#include "unicode-helper.h" #include "glbl.h" /* static data */ @@ -78,22 +83,19 @@ wtpGetDbgHdr(wtp_t *pThis) /* Not implemented dummy function for constructor */ -static rsRetVal NotImplementedDummy() { return RS_RET_OK; } +static rsRetVal NotImplementedDummy() { return RS_RET_NOT_IMPLEMENTED; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); - pthread_mutex_init(&pThis->mut, NULL); - pthread_mutex_init(&pThis->mutThrdShutdwn, NULL); + pthread_mutex_init(&pThis->mutWtp, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); + pthread_attr_init(&pThis->attrThrd); + pthread_attr_setdetachstate(&pThis->attrThrd, PTHREAD_CREATE_DETACHED); /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; - pThis->pfIsIdle = NotImplementedDummy; + pThis->pfGetDeqBatchSize = NotImplementedDummy; pThis->pfDoWork = NotImplementedDummy; - pThis->pfOnIdle = NotImplementedDummy; - pThis->pfOnWorkerCancel = NotImplementedDummy; - pThis->pfOnWorkerStartup = NotImplementedDummy; - pThis->pfOnWorkerShutdown = NotImplementedDummy; + pThis->pfObjProcessed = NotImplementedDummy; ENDobjConstruct(wtp) @@ -111,13 +113,12 @@ wtpConstructFinalize(wtp_t *pThis) ISOBJ_TYPE_assert(pThis, wtp); - dbgprintf("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis)); + DBGPRINTF("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis)); /* alloc and construct workers - this can only be done in finalizer as we previously do * not know the max number of workers */ - if((pThis->pWrkr = malloc(sizeof(wti_t*) * pThis->iNumWorkerThreads)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - + CHKmalloc(pThis->pWrkr = MALLOC(sizeof(wti_t*) * pThis->iNumWorkerThreads)); + for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { CHKiRet(wtiConstruct(&pThis->pWrkr[i])); pWti = pThis->pWrkr[i]; @@ -137,8 +138,6 @@ finalize_it: BEGINobjDestruct(wtp) /* be sure to specify the object type also in END and CODESTART macros! */ int i; CODESTARTobjDestruct(wtp) - wtpProcessThrdChanges(pThis); /* process thread changes one last time */ - /* destruct workers */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) wtiDestruct(&pThis->pWrkr[i]); @@ -148,137 +147,68 @@ CODESTARTobjDestruct(wtp) /* actual destruction */ pthread_cond_destroy(&pThis->condThrdTrm); - pthread_mutex_destroy(&pThis->mut); - pthread_mutex_destroy(&pThis->mutThrdShutdwn); + pthread_mutex_destroy(&pThis->mutWtp); + pthread_attr_destroy(&pThis->attrThrd); - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->pszDbgHdr); ENDobjDestruct(wtp) -/* wake up at least one worker thread. - * rgerhards, 2008-01-20 - */ -rsRetVal -wtpWakeupWrkr(wtp_t *pThis) -{ - DEFiRet; - - /* TODO; mutex? I think not needed, as we do not need predictable exec order -- rgerhards, 2008-01-28 */ - ISOBJ_TYPE_assert(pThis, wtp); - pthread_cond_signal(pThis->pcondBusy); - RETiRet; -} - -/* wake up all worker threads. - * rgerhards, 2008-01-16 - */ -rsRetVal -wtpWakeupAllWrkr(wtp_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, wtp); - d_pthread_mutex_lock(pThis->pmutUsr); - pthread_cond_broadcast(pThis->pcondBusy); - d_pthread_mutex_unlock(pThis->pmutUsr); - RETiRet; -} - - -/* check if we had any worker thread changes and, if so, act - * on them. At a minimum, terminated threads are harvested (joined). - * This function MUST NEVER block on the queue mutex! - */ -rsRetVal -wtpProcessThrdChanges(wtp_t *pThis) -{ - DEFiRet; - int i; - - ISOBJ_TYPE_assert(pThis, wtp); - - if(pThis->bThrdStateChanged == 0) - FINALIZE; - - if(d_pthread_mutex_trylock(&(pThis->mutThrdShutdwn)) != 0) { - /* another thread is already in the loop */ - FINALIZE; - } - - /* Note: there is a left-over potential race condition below: - * pThis->bThrdStateChanged may be re-set by another thread while - * we work on it and thus the loop may terminate too early. However, - * there are no really bad effects from that so I perfer - for this - * version - to live with the problem as is. Not a good idea to - * introduce that large change into the stable branch without very - * good reason. -- rgerhards, 2009-04-02 - */ - do { - /* reset the change marker */ - pThis->bThrdStateChanged = 0; - /* go through all threads */ - for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - wtiProcessThrdChanges(pThis->pWrkr[i], LOCK_MUTEX); - } - /* restart if another change occured while we were processing the changes */ - } while(pThis->bThrdStateChanged != 0); - - d_pthread_mutex_unlock(&(pThis->mutThrdShutdwn)); - -finalize_it: - RETiRet; -} - - -/* Sent a specific state for the worker thread pool. - * rgerhards, 2008-01-21 +/* Sent a specific state for the worker thread pool. -- rgerhards, 2008-01-21 + * We do not need to do atomic instructions as set operations are only + * called when terminating the pool, and then in strict sequence. So we + * can never overwrite each other. On the other hand, it also doesn't + * matter if the read operation obtains an older value, as we then simply + * do one more iteration, what is perfectly legal (during shutdown + * they are awoken in any case). -- rgerhards, 2009-07-20 */ rsRetVal wtpSetState(wtp_t *pThis, wtpState_t iNewState) { - DEFiRet; - ISOBJ_TYPE_assert(pThis, wtp); pThis->wtpState = iNewState; - /* TODO: must wakeup workers? seen to be not needed -- rgerhards, 2008-01-28 */ - - RETiRet; + return RS_RET_OK; } /* check if the worker shall shutdown (1 = yes, 0 = no) - * TODO: check if we can use atomic operations to enhance performance * Note: there may be two mutexes locked, the bLockUsrMutex is the one in our "user" * (e.g. the queue clas) * rgerhards, 2008-01-21 */ rsRetVal -wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex) +wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex) { DEFiRet; - DEFVARS_mutexProtection; + wtpState_t wtpState; ISOBJ_TYPE_assert(pThis, wtp); + /* we need a consistent value, but it doesn't really matter if it is changed + * right after the fetch - then we simply do one more iteration in the worker + */ + wtpState = ATOMIC_FETCH_32BIT(pThis->wtpState); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - if( (pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) - || ((pThis->wtpState == wtpState_SHUTDOWN) && pThis->pfIsIdle(pThis->pUsr, bLockUsrMutex))) - iRet = RS_RET_TERMINATE_NOW; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + if(wtpState == wtpState_SHUTDOWN_IMMEDIATE) { + ABORT_FINALIZE(RS_RET_TERMINATE_NOW); + } else if(wtpState == wtpState_SHUTDOWN) { + ABORT_FINALIZE(RS_RET_TERMINATE_WHEN_IDLE); + } /* try customer handler if one was set and we do not yet have a definite result */ - if(iRet == RS_RET_OK && pThis->pfChkStopWrkr != NULL) { + if(pThis->pfChkStopWrkr != NULL) { iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex); } +finalize_it: RETiRet; } #pragma GCC diagnostic ignored "-Wempty-body" /* Send a shutdown command to all workers and see if they terminate. - * A timeout may be specified. + * A timeout may be specified. This function may also be called with + * the current number of workers being 0, in which case it does not + * shut down any worker. * rgerhards, 2008-01-14 */ rsRetVal @@ -286,30 +216,25 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout { DEFiRet; int bTimedOut; - int iCancelStateSave; ISOBJ_TYPE_assert(pThis, wtp); + /* lock mutex to prevent races (may otherwise happen during idle processing and such...) */ + d_pthread_mutex_lock(pThis->pmutUsr); wtpSetState(pThis, tShutdownCmd); - wtpWakeupAllWrkr(pThis); + pthread_cond_broadcast(pThis->pcondBusy); /* wake up all workers */ + d_pthread_mutex_unlock(pThis->pmutUsr); - /* see if we need to harvest (join) any terminated threads (even in timeout case, - * some may have terminated... - */ - wtpProcessThrdChanges(pThis); - - /* and wait for their termination */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pThis->mut); - pthread_cleanup_push(mutexCancelCleanup, &pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); + /* wait for worker thread termination */ + d_pthread_mutex_lock(&pThis->mutWtp); + pthread_cleanup_push(mutexCancelCleanup, &pThis->mutWtp); bTimedOut = 0; while(pThis->iCurNumWrkThrd > 0 && !bTimedOut) { - dbgprintf("%s: waiting %ldms on worker thread termination, %d still running\n", - wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), pThis->iCurNumWrkThrd); + DBGPRINTF("%s: waiting %ldms on worker thread termination, %d still running\n", + wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd)); - if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mut, ptTimeout) != 0) { - dbgprintf("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis)); + if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mutWtp, ptTimeout) != 0) { + DBGPRINTF("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis)); bTimedOut = 1; /* we exit the loop on timeout */ } } @@ -318,40 +243,11 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout if(bTimedOut) iRet = RS_RET_TIMED_OUT; - /* see if we need to harvest (join) any terminated threads (even in timeout case, - * some may have terminated... - */ - wtpProcessThrdChanges(pThis); - RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" -/* indicate that a thread has terminated and awake anyone waiting on it - * rgerhards, 2008-01-23 - */ -rsRetVal wtpSignalWrkrTermination(wtp_t *pThis) -{ - DEFiRet; - /* I leave the mutex code here out as it gives us deadlocks. I think it is not really - * needed and we are on the safe side. I leave this comment in if practice proves us - * wrong. The whole thing should be removed after half a year or year if we see there - * actually is no issue (or revisit it from a theoretical POV). - * rgerhards, 2008-01-28 - * revisited 2008-09-30, still a bit unclear, leave in - */ - /*TODO: mutex or not mutex, that's the question ;)DEFVARS_mutexProtection;*/ - - ISOBJ_TYPE_assert(pThis, wtp); - - /*BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX);*/ - pthread_cond_signal(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ - /*END_MTX_PROTECTED_OPERATIONS(&pThis->mut);*/ - RETiRet; -} - - /* Unconditionally cancel all running worker threads. * rgerhards, 2008-01-14 */ @@ -363,12 +259,8 @@ wtpCancelAll(wtp_t *pThis) ISOBJ_TYPE_assert(pThis, wtp); - /* process any pending thread requests so that we know who actually is still running */ - wtpProcessThrdChanges(pThis); - /* go through all workers and cancel those that are active */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - dbgprintf("%s: try canceling worker thread %d\n", wtpGetDbgHdr(pThis), i); wtiCancelThrd(pThis->pWrkr[i]); } @@ -376,40 +268,56 @@ wtpCancelAll(wtp_t *pThis) } - -/* Set the Inactivity Guard - * rgerhards, 2008-01-21 +/* this function contains shared code for both regular worker shutdown as + * well as shutdown via cancellation. We can not simply use pthread_cleanup_pop(1) + * as this introduces a race in the debug system (RETiRet system). + * rgerhards, 2009-10-26 */ -rsRetVal -wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex) +static inline void +wtpWrkrExecCleanup(wti_t *pWti) { - DEFiRet; - DEFVARS_mutexProtection; + wtp_t *pThis; - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - pThis->bInactivityGuard = bNewState; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + BEGINfunc + ISOBJ_TYPE_assert(pWti, wti); + pThis = pWti->pWtp; + ISOBJ_TYPE_assert(pThis, wtp); - RETiRet; + /* the order of the next two statements is important! */ + wtiSetState(pWti, WRKTHRD_STOPPED); + ATOMIC_DEC(pThis->iCurNumWrkThrd); + + DBGPRINTF("%s: Worker thread %lx, terminated, num workers now %d\n", + wtpGetDbgHdr(pThis), (unsigned long) pWti, ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd)); + + ENDfunc } -/* cancellation cleanup handler for executing worker - * decrements the worker counter - * rgerhards, 2008-01-20 +/* cancellation cleanup handler for executing worker decrements the worker counter. + * rgerhards, 2009-07-20 */ -void +static void wtpWrkrExecCancelCleanup(void *arg) { - wtp_t *pThis = (wtp_t*) arg; + wti_t *pWti = (wti_t*) arg; + wtp_t *pThis; BEGINfunc + ISOBJ_TYPE_assert(pWti, wti); + pThis = pWti->pWtp; ISOBJ_TYPE_assert(pThis, wtp); - pThis->iCurNumWrkThrd--; - wtpSignalWrkrTermination(pThis); + DBGPRINTF("%s: Worker thread %lx requested to be cancelled.\n", + wtpGetDbgHdr(pThis), (unsigned long) pWti); + + wtpWrkrExecCleanup(pWti); - dbgprintf("%s: thread CANCELED with %d workers running.\n", wtpGetDbgHdr(pThis), pThis->iCurNumWrkThrd); ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ } @@ -421,12 +329,13 @@ wtpWrkrExecCancelCleanup(void *arg) static void * wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in wtp! */ { - DEFiRet; - DEFVARS_mutexProtection; + uchar *pszDbgHdr; + uchar thrdName[32] = "rs:"; wti_t *pWti = (wti_t*) arg; wtp_t *pThis; sigset_t sigSet; + BEGINfunc ISOBJ_TYPE_assert(pWti, wti); pThis = pWti->pWtp; ISOBJ_TYPE_assert(pThis, wtp); @@ -434,41 +343,26 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in sigfillset(&sigSet); pthread_sigmask(SIG_BLOCK, &sigSet, NULL); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - - /* do some late initialization */ - - pthread_cleanup_push(wtpWrkrExecCancelCleanup, pThis); - - /* finally change to RUNNING state. We need to check if we actually should still run, - * because someone may have requested us to shut down even before we got a chance to do - * our init. That would be a bad race... -- rgerhards, 2008-01-16 - */ - wtiSetState(pWti, eWRKTHRD_RUNNING, 0, MUTEX_ALREADY_LOCKED); /* we are running now! */ - - do { - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - iRet = wtiWorker(pWti); /* just to make sure: this is NOT protected by the mutex! */ - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - } while(pThis->iCurNumWrkThrd == 1 && pThis->bInactivityGuard == 1); - /* inactivity guard prevents shutdown of all workers while one should be running due to race - * condition. It can lead to one more worker running than desired, but that is acceptable. After - * all, that worker will shutdown itself due to inactivity timeout. If, however, none were running - * when one was required, processing could come to a halt. -- rgerhards, 2008-01-21 - */ +# if HAVE_PRCTL && defined PR_SET_NAME + /* set thread name - we ignore if the call fails, has no harsh consequences... */ + pszDbgHdr = wtpGetDbgHdr(pThis); + ustrncpy(thrdName+3, pszDbgHdr, 20); + if(prctl(PR_SET_NAME, thrdName, 0, 0, 0) != 0) { + DBGPRINTF("prctl failed, not setting thread name for '%s'\n", wtpGetDbgHdr(pThis)); + } +# endif + pthread_cleanup_push(wtpWrkrExecCancelCleanup, pWti); + wtiWorker(pWti); pthread_cleanup_pop(0); - pThis->iCurNumWrkThrd--; - wtpSignalWrkrTermination(pThis); - - dbgprintf("%s: Worker thread %lx, terminated, num workers now %d\n", - wtpGetDbgHdr(pThis), (unsigned long) pWti, pThis->iCurNumWrkThrd); - - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + wtpWrkrExecCleanup(pWti); ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ pthread_exit(0); } #pragma GCC diagnostic warning "-Wempty-body" @@ -476,27 +370,20 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in /* start a new worker */ static rsRetVal -wtpStartWrkr(wtp_t *pThis, int bLockMutex) +wtpStartWrkr(wtp_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; wti_t *pWti; int i; int iState; + DEFiRet; ISOBJ_TYPE_assert(pThis, wtp); - wtpProcessThrdChanges(pThis); // TODO: Performance: this causes a lot of FUTEX calls - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); + d_pthread_mutex_lock(&pThis->mutWtp); - pThis->iCurNumWrkThrd++; - - /* find free spot in thread table. If we find at least one worker that is in initialization, - * we do NOT start a new one. Let's give the other one a chance, first. - */ + /* find free spot in thread table. */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - if(wtiGetState(pThis->pWrkr[i], LOCK_MUTEX) == eWRKTHRD_STOPPED) { + if(wtiGetState(pThis->pWrkr[i]) == WRKTHRD_STOPPED) { break; } } @@ -504,25 +391,20 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) if(i == pThis->iNumWorkerThreads) ABORT_FINALIZE(RS_RET_NO_MORE_THREADS); + if(i == 0 || pThis->toWrkShutdown == -1) { + wtiSetAlwaysRunning(pThis->pWrkr[i]); + } + pWti = pThis->pWrkr[i]; - wtiSetState(pWti, eWRKTHRD_RUN_CREATED, 0, LOCK_MUTEX); - iState = pthread_create(&(pWti->thrdID), NULL, wtpWorker, (void*) pWti); - dbgprintf("%s: started with state %d, num workers now %d\n", - wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd); - - /* we try to give the starting worker a little boost. It won't help much as we still - * hold the queue's mutex, but at least it has a chance to start on a single-CPU system. - */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); -# endif + wtiSetState(pWti, WRKTHRD_RUNNING); + iState = pthread_create(&(pWti->thrdID), &pThis->attrThrd, wtpWorker, (void*) pWti); + ATOMIC_INC(pThis->iCurNumWrkThrd); /* we got one more! */ - /* indicate we just started a worker and would like to see it running */ - wtpSetInactivityGuard(pThis, 1, MUTEX_ALREADY_LOCKED); + DBGPRINTF("%s: started with state %d, num workers now %d\n", + wtpGetDbgHdr(pThis), iState, ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd)); finalize_it: - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + d_pthread_mutex_unlock(&pThis->mutWtp); RETiRet; } @@ -539,7 +421,6 @@ rsRetVal wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) { DEFiRet; - DEFVARS_mutexProtection; int nMissing; /* number workers missing to run */ int i; @@ -548,29 +429,24 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) if(nMaxWrkr == 0) FINALIZE; - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - if(nMaxWrkr > pThis->iNumWorkerThreads) /* limit to configured maximum */ nMaxWrkr = pThis->iNumWorkerThreads; - nMissing = nMaxWrkr - pThis->iCurNumWrkThrd; + nMissing = nMaxWrkr - ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd); if(nMissing > 0) { - dbgprintf("%s: high activity - starting %d additional worker thread(s).\n", wtpGetDbgHdr(pThis), nMissing); + DBGPRINTF("%s: high activity - starting %d additional worker thread(s).\n", + wtpGetDbgHdr(pThis), nMissing); /* start the rqtd nbr of workers */ for(i = 0 ; i < nMissing ; ++i) { - CHKiRet(wtpStartWrkr(pThis, MUTEX_ALREADY_LOCKED)); - } - } else { - if(nMaxWrkr > 0) { - dbgprintf("wtpAdviseMaxWorkers signals busy\n"); - wtpWakeupWrkr(pThis); + CHKiRet(wtpStartWrkr(pThis)); } + } else { + pthread_cond_signal(pThis->pcondBusy); } finalize_it: - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); RETiRet; } @@ -584,35 +460,9 @@ DEFpropSetMethPTR(wtp, pmutUsr, pthread_mutex_t) DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t) DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)) -DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)) -DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) -DEFpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*, void*)) -DEFpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)) - - -/* return the current number of worker threads. - * TODO: atomic operation would bring a nice performance - * enhancemcent - * rgerhards, 2008-01-27 - */ -int -wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex) -{ - DEFVARS_mutexProtection; - int iNumWrkr; - - BEGINfunc - ISOBJ_TYPE_assert(pThis, wtp); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - iNumWrkr = pThis->iCurNumWrkThrd; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - ENDfunc - return iNumWrkr; -} +DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)) +DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)) +DEFpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)) /* set the debug header message @@ -636,7 +486,7 @@ wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg) pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ @@ -666,6 +516,5 @@ BEGINObjClassInit(wtp, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wtp) -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/wtp.h b/runtime/wtp.h index b9cb07c5..05c02a8c 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -27,18 +27,9 @@ #include <pthread.h> #include "obj.h" -/* commands and states for worker threads. */ -typedef enum { - eWRKTHRD_STOPPED = 0, /* worker thread is not running (either actually never ran or was shut down) */ - eWRKTHRD_TERMINATING = 1,/* worker thread has shut down, but some finalzing is still needed */ - /* ALL active states MUST be numerically higher than eWRKTHRD_TERMINATED and NONE must be lower! */ - eWRKTHRD_RUN_CREATED = 2,/* worker thread has been created, but not yet begun initialization (prob. not yet scheduled) */ - eWRKTHRD_RUN_INIT = 3, /* worker thread is initializing, but not yet fully running */ - eWRKTHRD_RUNNING = 4, /* worker thread is up and running and shall continue to do so */ - eWRKTHRD_SHUTDOWN = 5, /* worker thread is running but shall terminate when wtp is empty */ - eWRKTHRD_SHUTDOWN_IMMEDIATE = 6/* worker thread is running but shall terminate even if wtp is full */ - /* SHUTDOWN_IMMEDIATE MUST alsways be the numerically highest state! */ -} qWrkCmd_t; +/* states for worker threads. */ +#define WRKTHRD_STOPPED FALSE +#define WRKTHRD_RUNNING TRUE /* possible states of a worker thread pool */ @@ -50,37 +41,31 @@ typedef enum { /* the worker thread pool (wtp) object */ -typedef struct wtp_s { +struct wtp_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ wtpState_t wtpState; int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ struct wti_s **pWrkr;/* array with control structure for the worker thread(s) associated with this wtp */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ - int bInactivityGuard;/* prevents inactivity due to race condition */ rsRetVal (*pConsumer)(void *); /* user-supplied consumer function for dewtpd messages */ /* synchronization variables */ - pthread_mutex_t mutThrdShutdwn; /* mutex to guard thread shutdown processing */ - pthread_mutex_t mut; /* mutex for the wtp's thread management */ + pthread_mutex_t mutWtp; /* mutex for the wtp's thread management */ pthread_cond_t condThrdTrm;/* signalled when threads terminate */ - int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* user objects */ - void *pUsr; /* pointer to user object */ + void *pUsr; /* pointer to user object (in this case, the queue the wtp belongs to) */ + pthread_attr_t attrThrd;/* attribute for new threads (created just once and cached here) */ pthread_mutex_t *pmutUsr; pthread_cond_t *pcondBusy; /* condition the user will signal "busy again, keep runing" on (awakes worker) */ rsRetVal (*pfChkStopWrkr)(void *pUsr, int); + rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */ + rsRetVal (*pfObjProcessed)(void *pUsr, wti_t *pWti); /* indicate user object is processed */ rsRetVal (*pfRateLimiter)(void *pUsr); - rsRetVal (*pfIsIdle)(void *pUsr, int); - rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int); - rsRetVal (*pfOnIdle)(void *pUsr, int); - rsRetVal (*pfOnWorkerCancel)(void *pUsr, void*pWti); - rsRetVal (*pfOnWorkerStartup)(void *pUsr); - rsRetVal (*pfOnWorkerShutdown)(void *pUsr); + rsRetVal (*pfDoWork)(void *pUsr, void *pWti); /* end user objects */ uchar *pszDbgHdr; /* header string for debug messages */ -} wtp_t; +}; /* some symbolic constants for easier reference */ @@ -91,25 +76,18 @@ rsRetVal wtpConstructFinalize(wtp_t *pThis); rsRetVal wtpDestruct(wtp_t **ppThis); rsRetVal wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr); rsRetVal wtpProcessThrdChanges(wtp_t *pThis); -rsRetVal wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex); -rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex); +rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex); rsRetVal wtpSetState(wtp_t *pThis, wtpState_t iNewState); -rsRetVal wtpWakeupWrkr(wtp_t *pThis); rsRetVal wtpWakeupAllWrkr(wtp_t *pThis); rsRetVal wtpCancelAll(wtp_t *pThis); rsRetVal wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg); -rsRetVal wtpSignalWrkrTermination(wtp_t *pWtp); rsRetVal wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout); -int wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex); PROTOTYPEObjClassInit(wtp); PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*,void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)); +PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)); +PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)); +PROTOTYPEpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)); PROTOTYPEpropSetMeth(wtp, toWrkShutdown, long); PROTOTYPEpropSetMeth(wtp, wtpState, wtpState_t); PROTOTYPEpropSetMeth(wtp, iMaxWorkerThreads, int); diff --git a/runtime/zlibw.c b/runtime/zlibw.c new file mode 100644 index 00000000..2b386213 --- /dev/null +++ b/runtime/zlibw.c @@ -0,0 +1,125 @@ +/* The zlibwrap object. + * + * This is an rsyslog object wrapper around zlib. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include <string.h> +#include <assert.h> +#include <zlib.h> + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "zlibw.h" + +MODULE_TYPE_LIB + +/* static data */ +DEFobjStaticHelpers + + +/* ------------------------------ methods ------------------------------ */ + +/* zlib make strong use of macros for its interface functions, so we can not simply + * pass function pointers to them. Instead, we create very small wrappers which call + * the relevant entry points. + */ + +static int myDeflateInit(z_streamp strm, int level) +{ + return deflateInit(strm, level); +} + +static int myDeflateInit2(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy) +{ + return deflateInit2(strm, level, method, windowBits, memLevel, strategy); +} + +static int myDeflateEnd(z_streamp strm) +{ + return deflateEnd(strm); +} + +static int myDeflate(z_streamp strm, int flush) +{ + return deflate(strm, flush); +} + + +/* queryInterface function + * rgerhards, 2008-03-05 + */ +BEGINobjQueryInterface(zlibw) +CODESTARTobjQueryInterface(zlibw) + if(pIf->ifVersion != zlibwCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DeflateInit = myDeflateInit; + pIf->DeflateInit2 = myDeflateInit2; + pIf->Deflate = myDeflate; + pIf->DeflateEnd = myDeflateEnd; +finalize_it: +ENDobjQueryInterface(zlibw) + + +/* Initialize the zlibw class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINAbstractObjClassInit(zlibw, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ + /* request objects we use */ + + /* set our own handlers */ +ENDObjClassInit(zlibw) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + CHKiRet(zlibwClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + /* Initialize all classes that are in our module - this includes ourselfs */ +ENDmodInit +/* vi:set ai: + */ diff --git a/runtime/zlibw.h b/runtime/zlibw.h new file mode 100644 index 00000000..63d8f386 --- /dev/null +++ b/runtime/zlibw.h @@ -0,0 +1,46 @@ +/* The zlibw object. It encapsulates the zlib functionality. The primary + * purpose of this wrapper class is to enable rsyslogd core to be build without + * zlib libraries. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_ZLIBW_H +#define INCLUDED_ZLIBW_H + +#include <zlib.h> + +/* interfaces */ +BEGINinterface(zlibw) /* name must also be changed in ENDinterface macro! */ + int (*DeflateInit)(z_streamp strm, int); + int (*DeflateInit2)(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy); + int (*Deflate)(z_streamp strm, int); + int (*DeflateEnd)(z_streamp strm); +ENDinterface(zlibw) +#define zlibwCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(zlibw); + +/* the name of our library binary */ +#define LM_ZLIBW_FILENAME "lmzlibw" + +#endif /* #ifndef INCLUDED_ZLIBW_H */ diff --git a/shave-libtool.in b/shave-libtool.in deleted file mode 100644 index 54ebd690..00000000 --- a/shave-libtool.in +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2009, Damien Lespiau <damien.lespiau@gmail.com> -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -# we need sed -SED=@SED@ -if test -z "$SED" ; then -SED=sed -fi - -lt_unmangle () -{ - last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` -} - -# the real libtool to use -LIBTOOL="$1" -shift - -# if 1, don't print anything, the underlaying wrapper will do it -pass_though=0 - -# scan the arguments, keep the right ones for libtool, and discover the mode -preserved_args= - -# have we seen the --tag option of libtool in the command line ? -tag_seen=0 - -while test "$#" -gt 0; do - opt="$1" - shift - - case $opt in - --mode=*) - mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` - preserved_args="$preserved_args $opt" - ;; - -o) - lt_output="$1" - preserved_args="$preserved_args $opt" - ;; - --tag=*) - tag_seen=1 - preserved_args="$preserved_args $opt" - ;; - *) - preserved_args="$preserved_args $opt" - ;; - esac -done - -case "$mode" in -compile) - # shave will be called and print the actual CC/CXX/LINK line - preserved_args="$preserved_args --shave-mode=$mode" - pass_though=1 - ;; -link) - preserved_args="$preserved_args --shave-mode=$mode" - Q=" LINK " - ;; -*) - # let's u - # echo "*** libtool: Unimplemented mode: $mode, fill a bug report" - ;; -esac - -lt_unmangle "$lt_output" -output=$last_result - -# automake does not add a --tag switch to its libtool invocation when -# assembling a .s file and rely on libtool to infer the right action based -# on the compiler name. As shave is using CC to hook a wrapper, libtool gets -# confused. Let's detect these cases and add a --tag=CC option. -tag="" -if test $tag_seen -eq 0 -a x"$mode" = xcompile; then - tag="--tag=CC" -fi - -if test -z $V; then - if test $pass_though -eq 0; then - echo "$Q$output" - fi - $LIBTOOL --silent $tag $preserved_args -else - echo $LIBTOOL $tag $preserved_args - $LIBTOOL $tag $preserved_args -fi diff --git a/shave.in b/shave.in deleted file mode 100644 index afed42e1..00000000 --- a/shave.in +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2009, Damien Lespiau <damien.lespiau@gmail.com> -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -# we need sed -SED=@SED@ -if test -z "$SED" ; then -SED=sed -fi - -lt_unmangle () -{ - last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` -} - -# the tool to wrap (cc, cxx, ar, ranlib, ..) -tool="$1" -shift - -# the reel tool (to call) -REEL_TOOL="$1" -shift - -pass_through=0 -preserved_args= -while test "$#" -gt 0; do - opt="$1" - shift - - case $opt in - --shave-mode=*) - mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` - ;; - -o) - lt_output="$1" - preserved_args="$preserved_args $opt" - ;; - *) - preserved_args="$preserved_args $opt" - ;; - esac -done - -# mode=link is handled in the libtool wrapper -case "$mode,$tool" in -link,*) - pass_through=1 - ;; -*,cxx) - Q=" CXX " - ;; -*,cc) - Q=" CC " - ;; -*,fc) - Q=" FC " - ;; -*,f77) - Q=" F77 " - ;; -*,objc) - Q=" OBJC " - ;; -*,*) - # should not happen - Q=" CC " - ;; -esac - -lt_unmangle "$lt_output" -output=$last_result - -if test -z $V; then - if test $pass_through -eq 0; then - echo "$Q$output" - fi - $REEL_TOOL $preserved_args -else - echo $REEL_TOOL $preserved_args - $REEL_TOOL $preserved_args -fi @@ -169,7 +169,7 @@ TCPSendBldFrame(tcpclt_t *pThis, char **pmsg, size_t *plen, int *pbMustBeFreed) * I have added this comment so that the logic is not accidently * changed again. rgerhards, 2005-10-25 */ - if((buf = malloc((len + 2) * sizeof(char))) == NULL) { + if((buf = MALLOC((len + 2) * sizeof(char))) == NULL) { /* extreme mem shortage, try to solve * as good as we can. No point in calling * any alarms, they might as well run out @@ -217,7 +217,7 @@ TCPSendBldFrame(tcpclt_t *pThis, char **pmsg, size_t *plen, int *pbMustBeFreed) /* IETF20061218 iLenBuf = snprintf(szLenBuf, sizeof(szLenBuf)/sizeof(char), "%d ", len + iLenBuf);*/ - if((buf = malloc((len + iLenBuf) * sizeof(char))) == NULL) { + if((buf = MALLOC((len + iLenBuf) * sizeof(char))) == NULL) { /* we are out of memory. This is an extreme situation. We do not * call any alarm handlers because they most likely run out of mem, * too. We are brave enough to call debug output, though. Other than @@ -297,6 +297,12 @@ Send(tcpclt_t *pThis, void *pData, char *msg, size_t len) CHKiRet(TCPSendBldFrame(pThis, &msg, &len, &bMsgMustBeFreed)); + if(pThis->iRebindInterval > 0 && ++pThis->iNumMsgs == pThis->iRebindInterval) { + /* we need to rebind, and use the retry logic for this*/ + CHKiRet(pThis->prepRetryFunc(pData)); /* try to recover */ + pThis->iNumMsgs = 0; + } + while(!bDone) { /* loop is broken when send succeeds or error occurs */ CHKiRet(pThis->initFunc(pData)); iRet = pThis->sendFunc(pData, msg, len); @@ -318,7 +324,7 @@ Send(tcpclt_t *pThis, void *pData, char *msg, size_t len) * happens is that we lose our message recovery buffer - anything else would * be worse, so don't try anything ;) -- rgerhards, 2008-03-12 */ - if((pThis->prevMsg = malloc(len)) != NULL) { + if((pThis->prevMsg = MALLOC(len)) != NULL) { memcpy(pThis->prevMsg, msg, len); pThis->lenPrevMsg = len; } @@ -388,6 +394,13 @@ SetFraming(tcpclt_t *pThis, TCPFRAMINGMODE framing) pThis->tcp_framing = framing; RETiRet; } +static rsRetVal +SetRebindInterval(tcpclt_t *pThis, int iRebindInterval) +{ + DEFiRet; + pThis->iRebindInterval = iRebindInterval; + RETiRet; +} /* Standard-Constructor @@ -445,6 +458,7 @@ CODESTARTobjQueryInterface(tcpclt) pIf->SetSendFrame = SetSendFrame; pIf->SetSendPrepRetry = SetSendPrepRetry; pIf->SetFraming = SetFraming; + pIf->SetRebindInterval = SetRebindInterval; finalize_it: ENDobjQueryInterface(tcpclt) @@ -36,6 +36,8 @@ typedef struct tcpclt_s { short bResendLastOnRecon; /* should the last message be resent on a successful reconnect? */ size_t lenPrevMsg; /* session specific callbacks */ + int iRebindInterval; /* how often should the send socket be rebound? */ + int iNumMsgs; /* number of messages during current "rebind session" */ rsRetVal (*initFunc)(void*); rsRetVal (*sendFunc)(void*, char*, size_t); rsRetVal (*prepRetryFunc)(void*); @@ -55,8 +57,10 @@ BEGINinterface(tcpclt) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetSendFrame)(tcpclt_t*, rsRetVal (*)(void*, char*, size_t)); rsRetVal (*SetSendPrepRetry)(tcpclt_t*, rsRetVal (*)(void*)); rsRetVal (*SetFraming)(tcpclt_t*, TCPFRAMINGMODE framing); + /* v3, 2009-07-14*/ + rsRetVal (*SetRebindInterval)(tcpclt_t*, int iRebindInterval); ENDinterface(tcpclt) -#define tcpcltCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define tcpcltCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/tcps_sess.c b/tcps_sess.c index 62d51f66..69b40ad0 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -36,6 +36,7 @@ #include "rsyslog.h" #include "dirty.h" +#include "unicode-helper.h" #include "module-template.h" #include "net.h" #include "tcpsrv.h" @@ -45,6 +46,8 @@ #include "netstrm.h" #include "msg.h" #include "datetime.h" +#include "prop.h" +#include "debug.h" /* static data */ @@ -52,10 +55,14 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) DEFobjCurrIf(netstrm) +DEFobjCurrIf(prop) DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ +static int iNbrTimeUsed = 0; /* how often has previous time been used so far? */ + + /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); @@ -66,7 +73,7 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ /* now allocate the message reception buffer */ - CHKmalloc(pThis->pMsg = (uchar*) malloc(sizeof(uchar) * iMaxLine + 1)); + CHKmalloc(pThis->pMsg = (uchar*) MALLOC(sizeof(uchar) * iMaxLine + 1)); finalize_it: ENDobjConstruct(tcps_sess) @@ -98,11 +105,10 @@ CODESTARTobjDestruct(tcps_sess) } /* now destruct our own properties */ if(pThis->fromHost != NULL) - free(pThis->fromHost); + CHKiRet(prop.Destruct(&pThis->fromHost)); if(pThis->fromHostIP != NULL) - free(pThis->fromHostIP); - if(pThis->pMsg != NULL) - free(pThis->pMsg); + CHKiRet(prop.Destruct(&pThis->fromHostIP)); + free(pThis->pMsg); ENDobjDestruct(tcps_sess) @@ -124,12 +130,13 @@ SetHost(tcps_sess_t *pThis, uchar *pszHost) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHost != NULL) { - free(pThis->fromHost); - } - - pThis->fromHost = pszHost; + if(pThis->fromHost == NULL) + CHKiRet(prop.Construct(&pThis->fromHost)); + + CHKiRet(prop.SetString(pThis->fromHost, pszHost, ustrlen(pszHost))); +finalize_it: + free(pszHost); /* we must free according to our (old) calling conventions */ RETiRet; } @@ -144,12 +151,13 @@ SetHostIP(tcps_sess_t *pThis, uchar *pszHostIP) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHostIP != NULL) { - free(pThis->fromHostIP); - } - - pThis->fromHostIP = pszHostIP; + if(pThis->fromHostIP == NULL) + CHKiRet(prop.Construct(&pThis->fromHostIP)); + + CHKiRet(prop.SetString(pThis->fromHostIP, pszHostIP, ustrlen(pszHostIP))); +finalize_it: + free(pszHostIP); RETiRet; } @@ -226,11 +234,9 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis) +defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) { msg_t *pMsg; - struct syslogTime stTime; - time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -240,22 +246,24 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } - //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); - //} /* we now create our own message object and submit it to the queue */ - CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); - /* first trim the buffer to what we have actually received */ - CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); - memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); - pMsg->iLenRawMsg = pThis->iMsg; - MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); + CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); + MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); + MsgSetInputName(pMsg, pThis->pLstnInfo->pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pThis->fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); - CHKiRet(submitMsg(pMsg)); + MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); + + if(pMultiSub == NULL) { + CHKiRet(submitMsg(pMsg)); + } else { + pMultiSub->ppMsgs[pMultiSub->nElem++] = pMsg; + if(pMultiSub->nElem == pMultiSub->maxElem) + CHKiRet(multiSubmitMsg(pMultiSub)); + } + finalize_it: /* reset status variables */ @@ -281,6 +289,8 @@ finalize_it: static rsRetVal PrepareClose(tcps_sess_t *pThis) { + struct syslogTime stTime; + time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -306,7 +316,8 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - defaultDoSubmitMessage(pThis); + datetime.getCurrTime(&stTime, &ttGenTime); + defaultDoSubmitMessage(pThis, &stTime, ttGenTime, NULL); } finalize_it: @@ -325,10 +336,11 @@ Close(tcps_sess_t *pThis) ISOBJ_TYPE_assert(pThis, tcps_sess); netstrm.Destruct(&pThis->pStrm); - free(pThis->fromHost); - pThis->fromHost = NULL; /* not really needed, but... */ - free(pThis->fromHostIP); - pThis->fromHostIP = NULL; /* not really needed, but... */ + if(pThis->fromHost != NULL) { + prop.Destruct(&pThis->fromHost); + } + if(pThis->fromHostIP != NULL) + prop.Destruct(&pThis->fromHostIP); RETiRet; } @@ -341,7 +353,7 @@ Close(tcps_sess_t *pThis) * rgerhards, 2008-03-14 */ static rsRetVal -processDataRcvd(tcps_sess_t *pThis, char c) +processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -387,7 +399,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -395,10 +407,10 @@ processDataRcvd(tcps_sess_t *pThis, char c) */ } - if(( (c == '\n') + if(( ((c == '\n') && !pThis->pSrv->bDisableLFDelim) || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -415,7 +427,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); pThis->inputState = eAtStrtFram; } } @@ -436,27 +448,45 @@ processDataRcvd(tcps_sess_t *pThis, char c) * RS_RET_OK, which means the session should be kept open * or anything else, which means it must be closed. * rgerhards, 2008-03-01 + * As a performance optimization, we pick up the timestamp here. Acutally, + * this *is* the *correct* reception step for all the data we received, because + * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ +#define NUM_MULTISUB 1024 static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { - DEFiRet; + multi_submit_t multiSub; + msg_t *pMsgs[NUM_MULTISUB]; + struct syslogTime stTime; + time_t ttGenTime; char *pEnd; + DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); assert(pData != NULL); assert(iLen > 0); + datetime.getCurrTime(&stTime, &ttGenTime); + multiSub.ppMsgs = pMsgs; + multiSub.maxElem = NUM_MULTISUB; + multiSub.nElem = 0; + /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ + iNbrTimeUsed = 0; /* full time query */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++)); + CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime, &multiSub)); } + /* submit anything that was not yet submitted */ + CHKiRet(multiSubmitMsg(&multiSub)); + finalize_it: RETiRet; } +#undef NUM_MULTISUB /* queryInterface function @@ -503,6 +533,7 @@ CODESTARTObjClassExit(tcps_sess) objRelease(errmsg, CORE_COMPONENT); objRelease(netstrm, LM_NETSTRMS_FILENAME); objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDObjClassExit(tcps_sess) @@ -515,6 +546,7 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ @@ -525,7 +557,5 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, tcps_sessConstructFinalize); ENDObjClassInit(tcps_sess) - - /* vim:set ai: */ diff --git a/tcps_sess.h b/tcps_sess.h index 5e59aaab..ec3a6af4 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -24,6 +24,7 @@ #define INCLUDED_TCPS_SESS_H #include "obj.h" +#include "prop.h" /* a forward-definition, we are somewhat cyclic */ struct tcpsrv_s; @@ -44,8 +45,8 @@ struct tcps_sess_s { int iOctetsRemain; /* Number of Octets remaining in message */ TCPFRAMINGMODE eFraming; uchar *pMsg; /* message (fragment) received */ - uchar *fromHost; - uchar *fromHostIP; + prop_t *fromHost; /* host name we received messages from */ + prop_t *fromHostIP; void *pUsr; /* a user-pointer */ rsRetVal (*DoSubmitMessage)(tcps_sess_t*, uchar*, int); /* submit message callback */ }; @@ -1,8 +1,11 @@ /* tcpsrv.c * - * Common code for plain TCP based servers. This is currently being + * Common code for plain TCP syslog based servers. This is currently being * utilized by imtcp and imgssapi. * + * NOTE: this is *not* a generic TCP server, but one for syslog servers. For + * generic stream servers, please use ./runtime/strmsrv.c! + * * There are actually two classes within the tcpserver code: one is * the tcpsrv itself, the other one is its sessions. This is a helper * class to tcpsrv. @@ -12,12 +15,9 @@ * callbacks before the code is run. The tcpsrv then calls back * into the specific input modules at the appropriate time. * - * NOTE: read comments in module-template.h to understand how this file - * works! - * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -65,7 +65,9 @@ #include "netstrms.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "errmsg.h" +#include "ruleset.h" #include "unicode-helper.h" MODULE_TYPE_LIB @@ -78,12 +80,15 @@ MODULE_TYPE_LIB DEFobjStaticHelpers DEFobjCurrIf(conf) DEFobjCurrIf(glbl) +DEFobjCurrIf(ruleset) DEFobjCurrIf(tcps_sess) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) +DEFobjCurrIf(nspoll) +DEFobjCurrIf(prop) /* add new listener port to listener port list @@ -97,12 +102,16 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) ISOBJ_TYPE_assert(pThis, tcpsrv); -dbgprintf("XXX: tcpsrv.c add port %s, name '%s'\n", pszPort, pThis->pszInputName); /* create entry */ - CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); + CHKmalloc(pEntry = MALLOC(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; - CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + pEntry->pRuleset = pThis->pRuleset; + + /* we need to create a property */ + CHKiRet(prop.Construct(&pEntry->pInputName)); + CHKiRet(prop.SetString(pEntry->pInputName, pThis->pszInputName, ustrlen(pThis->pszInputName))); + CHKiRet(prop.ConstructFinalize(pEntry->pInputName)); /* and add to list */ pEntry->pNext = pThis->pLstnPorts; @@ -155,9 +164,9 @@ TCPSessTblInit(tcpsrv_t *pThis) ISOBJ_TYPE_assert(pThis, tcpsrv); assert(pThis->pSessions == NULL); - dbgprintf("Allocating buffer for %d TCP sessions.\n", pThis->iSessMax); + DBGPRINTF("Allocating buffer for %d TCP sessions.\n", pThis->iSessMax); if((pThis->pSessions = (tcps_sess_t **) calloc(pThis->iSessMax, sizeof(tcps_sess_t *))) == NULL) { - dbgprintf("Error: TCPSessInit() could not alloc memory for TCP session table.\n"); + DBGPRINTF("Error: TCPSessInit() could not alloc memory for TCP session table.\n"); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -228,11 +237,13 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) if(pThis->pSessions != NULL) { /* close all TCP connections! */ - i = TCPSessGetNxtSess(pThis, -1); - while(i != -1) { - tcps_sess.Destruct(&pThis->pSessions[i]); - /* now get next... */ - i = TCPSessGetNxtSess(pThis, i); + if(!pThis->bUsingEPoll) { + i = TCPSessGetNxtSess(pThis, -1); + while(i != -1) { + tcps_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = TCPSessGetNxtSess(pThis, i); + } } /* we are done with the session table - so get rid of it... */ @@ -244,14 +255,14 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) pEntry = pThis->pLstnPorts; while(pEntry != NULL) { free(pEntry->pszPort); - free(pEntry->pszInputName); + prop.Destruct(&pEntry->pInputName); pDel = pEntry; pEntry = pEntry->pNext; free(pDel); } /* finally close our listen streams */ - for(i = 0 ; i < pThis->iLstnMax ; ++i) { + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { netstrm.Destruct(pThis->ppLstn + i); } } @@ -267,16 +278,15 @@ addTcpLstn(void *pUsr, netstrm_t *pLstn) tcpsrv_t *pThis = pPortList->pSrv; DEFiRet; -dbgprintf("XXX: addTcpLst name %s\n", pPortList->pszInputName); ISOBJ_TYPE_assert(pThis, tcpsrv); ISOBJ_TYPE_assert(pLstn, netstrm); - if(pThis->iLstnMax >= TCPLSTN_MAX_DEFAULT) + if(pThis->iLstnCurr >= pThis->iLstnMax) ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); - pThis->ppLstn[pThis->iLstnMax] = pLstn; - pThis->ppLstnPort[pThis->iLstnMax] = pPortList; - ++pThis->iLstnMax; + pThis->ppLstn[pThis->iLstnCurr] = pLstn; + pThis->ppLstnPort[pThis->iLstnCurr] = pPortList; + ++pThis->iLstnCurr; finalize_it: RETiRet; @@ -318,16 +328,19 @@ finalize_it: static rsRetVal create_tcp_socket(tcpsrv_t *pThis) { - tcpLstnPortList_t *pEntry; DEFiRet; + rsRetVal localRet; + tcpLstnPortList_t *pEntry; ISOBJ_TYPE_assert(pThis, tcpsrv); /* init all configured ports */ pEntry = pThis->pLstnPorts; while(pEntry != NULL) { -dbgprintf("XXX: tcpsrv.c create_tcp_socket do port %s\n", pEntry->pszPort); - CHKiRet(initTCPListener(pThis, pEntry)); + localRet = initTCPListener(pThis, pEntry); + if(localRet != RS_RET_OK) { + errmsg.LogError(0, localRet, "Could not create tcp listener, ignoring port %s.", pEntry->pszPort); + } pEntry = pEntry->pNext; } @@ -400,7 +413,7 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, * rgerhards, 2005-09-26 */ if(!pThis->pIsPermittedHost((struct sockaddr*) addr, (char*) fromHostFQDN, pThis->pUsr, pSess->pUsr)) { - dbgprintf("%s is not an allowed sender\n", fromHostFQDN); + DBGPRINTF("%s is not an allowed sender\n", fromHostFQDN); if(glbl.GetOption_DisallowWarning()) { errno = 0; errmsg.LogError(0, RS_RET_HOST_NOT_PERMITTED, "TCP message from disallowed sender %s discarded", fromHostFQDN); @@ -426,7 +439,8 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, } *ppSess = pSess; - pThis->pSessions[iSess] = pSess; + if(!pThis->bUsingEPoll) + pThis->pSessions[iSess] = pSess; pSess = NULL; /* this is now also handed over */ finalize_it: @@ -453,10 +467,85 @@ RunCancelCleanup(void *arg) } -/* This function is called to gather input. */ -#pragma GCC diagnostic ignored "-Wempty-body" +/* helper to close a session. Takes status of poll vs. select into consideration. + * rgerhards, 2009-11-25 + */ +static inline rsRetVal +closeSess(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) { + DEFiRet; + if(pPoll != NULL) { + CHKiRet(nspoll.Ctl(pPoll, (*ppSess)->pStrm, 0, *ppSess, NSDPOLL_IN, NSDPOLL_DEL)); + } + pThis->pOnRegularClose(*ppSess); + tcps_sess.Destruct(ppSess); +finalize_it: + RETiRet; +} + + +/* process a receive request on one of the streams + * If pPoll is non-NULL, we have a netstream in epoll mode, which means we need + * to remove any descriptor we close from the epoll set. + * rgerhards, 2009-07-020 + */ static rsRetVal -Run(tcpsrv_t *pThis) +doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) +{ + char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ + ssize_t iRcvd; + rsRetVal localRet; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + DBGPRINTF("netstream %p with new data\n", (*ppSess)->pStrm); + /* Receive message */ + iRet = pThis->pRcvData(*ppSess, buf, sizeof(buf), &iRcvd); + switch(iRet) { + case RS_RET_CLOSED: + if(pThis->bEmitMsgOnClose) { + uchar *pszPeer; + int lenPeer; + errno = 0; + prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); + errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "Netstream session %p closed by remote peer %s.\n", + (*ppSess)->pStrm, pszPeer); + } + CHKiRet(closeSess(pThis, ppSess, pPoll)); + break; + case RS_RET_RETRY: + /* we simply ignore retry - this is not an error, but we also have not received anything */ + break; + case RS_RET_OK: + /* valid data received, process it! */ + if((localRet = tcps_sess.DataRcvd(*ppSess, buf, iRcvd)) != RS_RET_OK) { + /* in this case, something went awfully wrong. + * We are instructed to terminate the session. + */ + errmsg.LogError(0, localRet, "Tearing down TCP Session - see " + "previous messages for reason(s)\n"); + CHKiRet(closeSess(pThis, ppSess, pPoll)); + } + break; + default: + errno = 0; + errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", + (*ppSess)->pStrm); + CHKiRet(closeSess(pThis, ppSess, pPoll)); + break; + } + +finalize_it: + RETiRet; +} + + +/* This function is called to gather input. + * This variant here is only used if we need to work with a netstream driver + * that does not support epoll(). + */ +#pragma GCC diagnostic ignored "-Wempty-body" +static inline rsRetVal +RunSelect(tcpsrv_t *pThis) { DEFiRet; int nfds; @@ -464,14 +553,13 @@ Run(tcpsrv_t *pThis) int iTCPSess; int bIsReady; tcps_sess_t *pNewSess; - nssel_t *pSel; - ssize_t iRcvd; + nssel_t *pSel = NULL; ISOBJ_TYPE_assert(pThis, tcpsrv); /* this is an endless loop - it is terminated by the framework canelling * this thread. Thus, we also need to instantiate a cancel cleanup handler - * to prevent us from leaking anything. -- rgerharsd, 20080-04-24 + * to prevent us from leaking anything. -- rgerhards, 20080-04-24 */ pthread_cleanup_push(RunCancelCleanup, (void*) &pSel); while(1) { @@ -480,7 +568,7 @@ Run(tcpsrv_t *pThis) CHKiRet(nssel.ConstructFinalize(pSel)); /* Add the TCP listen sockets to the list of read descriptors. */ - for(i = 0 ; i < pThis->iLstnMax ; ++i) { + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { CHKiRet(nssel.Add(pSel, pThis->ppLstn[i], NSDSEL_RD)); } @@ -495,11 +583,15 @@ Run(tcpsrv_t *pThis) /* wait for io to become ready */ CHKiRet(nssel.Wait(pSel, &nfds)); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ - for(i = 0 ; i < pThis->iLstnMax ; ++i) { + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + if(glbl.GetGlobalInputTermState() == 1) + ABORT_FINALIZE(RS_RET_FORCE_TERM); CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); + DBGPRINTF("New connect on NSD %p.\n", pThis->ppLstn[i]); SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); --nfds; /* indicate we have processed one */ } @@ -508,41 +600,11 @@ Run(tcpsrv_t *pThis) /* now check the sessions */ iTCPSess = TCPSessGetNxtSess(pThis, -1); while(nfds && iTCPSess != -1) { + if(glbl.GetGlobalInputTermState() == 1) + ABORT_FINALIZE(RS_RET_FORCE_TERM); CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ - dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); - - /* Receive message */ - iRet = pThis->pRcvData(pThis->pSessions[iTCPSess], buf, sizeof(buf), &iRcvd); - switch(iRet) { - case RS_RET_CLOSED: - pThis->pOnRegularClose(pThis->pSessions[iTCPSess]); - tcps_sess.Destruct(&pThis->pSessions[iTCPSess]); - break; - case RS_RET_RETRY: - /* we simply ignore retry - this is not an error, but we also have not received anything */ - break; - case RS_RET_OK: - /* valid data received, process it! */ - if(tcps_sess.DataRcvd(pThis->pSessions[iTCPSess], buf, iRcvd) != RS_RET_OK) { - /* in this case, something went awfully wrong. - * We are instructed to terminate the session. - */ - errmsg.LogError(0, NO_ERRCODE, "Tearing down TCP Session %d - see " - "previous messages for reason(s)\n", iTCPSess); - pThis->pOnErrClose(pThis->pSessions[iTCPSess]); - tcps_sess.Destruct(&pThis->pSessions[iTCPSess]); - } - break; - default: - errno = 0; - errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", - pThis->pSessions[iTCPSess]->pStrm); - pThis->pOnErrClose(pThis->pSessions[iTCPSess]); - tcps_sess.Destruct(&pThis->pSessions[iTCPSess]); - break; - } + doReceive(pThis, &pThis->pSessions[iTCPSess], NULL); --nfds; /* indicate we have processed one */ } iTCPSess = TCPSessGetNxtSess(pThis, iTCPSess); @@ -558,17 +620,100 @@ finalize_it: /* this is a very special case - this time only we do not exit the } /* note that this point is usually not reached */ - pthread_cleanup_pop(0); /* remove cleanup handler */ + pthread_cleanup_pop(1); /* remove cleanup handler */ RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" +/* This function is called to gather input. It tries doing that via the epoll() + * interface. If the driver does not support that, it falls back to calling its + * select() equivalent. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Run(tcpsrv_t *pThis) +{ + DEFiRet; + int i; + tcps_sess_t *pNewSess; + nspoll_t *pPoll = NULL; + void *pUsr; + rsRetVal localRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerhards, 20080-04-24 + */ + if((localRet = nspoll.Construct(&pPoll)) == RS_RET_OK) { + // TODO: set driver + localRet = nspoll.ConstructFinalize(pPoll); + } + if(localRet != RS_RET_OK) { + /* fall back to select */ + dbgprintf("tcpsrv could not use epoll() interface, iRet=%d, using select()\n", localRet); + iRet = RunSelect(pThis); + FINALIZE; + } + + dbgprintf("tcpsrv uses epoll() interface, nsdpol driver found\n"); + + /* flag that we are in epoll mode */ + pThis->bUsingEPoll = TRUE; + + /* Add the TCP listen sockets to the list of sockets to monitor */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + dbgprintf("Trying to add listener %d, pUsr=%p\n", i, pThis->ppLstn); + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_ADD)); + dbgprintf("Added listener %d\n", i); + } + + while(1) { + localRet = nspoll.Wait(pPoll, -1, &i, &pUsr); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + + /* check if we need to ignore the i/o ready state. We do this if we got an invalid + * return state. Validly, this can happen for RS_RET_EINTR, for other cases it may + * not be the right thing, but what is the right thing is really hard at this point... + */ + if(localRet != RS_RET_OK) + continue; + + dbgprintf("poll returned with i %d, pUsr %p\n", i, pUsr); + + if(pUsr == pThis->ppLstn) { + DBGPRINTF("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + CHKiRet(nspoll.Ctl(pPoll, pNewSess->pStrm, 0, pNewSess, NSDPOLL_IN, NSDPOLL_ADD)); + DBGPRINTF("New session created with NSD %p.\n", pNewSess); + } else { + pNewSess = (tcps_sess_t*) pUsr; + doReceive(pThis, &pNewSess, pPoll); + } + } + + /* remove the tcp listen sockets from the epoll set */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_DEL)); + } + +finalize_it: + if(pPoll != NULL) + nspoll.Destruct(&pPoll); + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ - pThis->iSessMax = TCPSESS_MAX_DEFAULT; /* TODO: useful default ;) */ + pThis->iSessMax = TCPSESS_MAX_DEFAULT; + pThis->iLstnMax = TCPLSTN_MAX_DEFAULT; pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + pThis->bDisableLFDelim = 0; pThis->OnMsgReceive = NULL; ENDobjConstruct(tcpsrv) @@ -591,8 +736,8 @@ tcpsrvConstructFinalize(tcpsrv_t *pThis) CHKiRet(netstrms.ConstructFinalize(pThis->pNS)); /* set up listeners */ - CHKmalloc(pThis->ppLstn = calloc(TCPLSTN_MAX_DEFAULT, sizeof(netstrm_t*))); - CHKmalloc(pThis->ppLstnPort = calloc(TCPLSTN_MAX_DEFAULT, sizeof(tcpLstnPortList_t*))); + CHKmalloc(pThis->ppLstn = calloc(pThis->iLstnMax, sizeof(netstrm_t*))); + CHKmalloc(pThis->ppLstnPort = calloc(pThis->iLstnMax, sizeof(tcpLstnPortList_t*))); iRet = pThis->OpenLstnSocks(pThis); finalize_it: @@ -725,6 +870,18 @@ SetOnMsgReceive(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, } +/* set enable/disable standard LF frame delimiter (use with care!) + * -- rgerhards, 2010-01-03 + */ +static rsRetVal +SetbDisableLFDelim(tcpsrv_t *pThis, int bVal) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->bDisableLFDelim = bVal; + RETiRet; +} + /* Set additional framing to use (if any) -- rgerhards, 2008-12-10 */ static rsRetVal @@ -755,6 +912,26 @@ finalize_it: } +/* Set the ruleset (ptr) to use */ +static rsRetVal +SetRuleset(tcpsrv_t *pThis, ruleset_t *pRuleset) +{ + DEFiRet; + pThis->pRuleset = pRuleset; + RETiRet; +} + + +/* Set connection close notification */ +static rsRetVal +SetNotificationOnRemoteClose(tcpsrv_t *pThis, int bNewVal) +{ + DEFiRet; + pThis->bEmitMsgOnClose = bNewVal; + RETiRet; +} + + /* here follows a number of methods that shuffle authentication settings down * to the drivers. Drivers not supporting these settings may return an error * state. @@ -799,6 +976,20 @@ SetDrvrPermPeers(tcpsrv_t *pThis, permittedPeers_t *pPermPeers) * -------------------------------------------------------------------------- */ +/* set max number of listeners + * this must be called before ConstructFinalize, or it will have no effect! + * rgerhards, 2009-08-17 + */ +static rsRetVal +SetLstnMax(tcpsrv_t *pThis, int iMax) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->iLstnMax = iMax; + RETiRet; +} + + /* set max number of sessions * this must be called before ConstructFinalize, or it will have no effect! * rgerhards, 2009-04-09 @@ -832,7 +1023,6 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->ConstructFinalize = tcpsrvConstructFinalize; pIf->Destruct = tcpsrvDestruct; - //pIf->SessAccept = SessAccept; pIf->configureTCPListen = configureTCPListen; pIf->create_tcp_socket = create_tcp_socket; pIf->Run = Run; @@ -840,7 +1030,9 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetUsrP = SetUsrP; pIf->SetInputName = SetInputName; pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; + pIf->SetbDisableLFDelim = SetbDisableLFDelim; pIf->SetSessMax = SetSessMax; + pIf->SetLstnMax = SetLstnMax; pIf->SetDrvrMode = SetDrvrMode; pIf->SetDrvrAuthMode = SetDrvrAuthMode; pIf->SetDrvrPermPeers = SetDrvrPermPeers; @@ -855,6 +1047,8 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetCBOnRegularClose = SetCBOnRegularClose; pIf->SetCBOnErrClose = SetCBOnErrClose; pIf->SetOnMsgReceive = SetOnMsgReceive; + pIf->SetRuleset = SetRuleset; + pIf->SetNotificationOnRemoteClose = SetNotificationOnRemoteClose; finalize_it: ENDobjQueryInterface(tcpsrv) @@ -868,6 +1062,8 @@ CODESTARTObjClassExit(tcpsrv) /* release objects we no longer need */ objRelease(tcps_sess, DONT_LOAD_LIB); objRelease(conf, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(netstrms, DONT_LOAD_LIB); @@ -888,9 +1084,12 @@ BEGINObjClassInit(tcpsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(nspoll, DONT_LOAD_LIB)); CHKiRet(objUse(tcps_sess, DONT_LOAD_LIB)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, tcpsrvDebugPrint); @@ -23,6 +23,7 @@ #define INCLUDED_TCPSRV_H #include "obj.h" +#include "prop.h" #include "tcps_sess.h" /* support for framing anomalies */ @@ -36,8 +37,9 @@ typedef enum ETCPsyslogFramingAnomaly { /* list of tcp listen ports */ struct tcpLstnPortList_s { uchar *pszPort; /**< the ports the listener shall listen on */ - uchar *pszInputName; /**< value to be used as input name */ + prop_t *pInputName; tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ + ruleset_t *pRuleset; /**< associated ruleset */ tcpLstnPortList_t *pNext; /**< next port or NULL */ }; @@ -50,13 +52,19 @@ struct tcpsrv_s { int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ uchar *pszInputName; /**< value to be used as input name */ + ruleset_t *pRuleset; /**< ruleset to bind to */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ - int iLstnMax; /**< max nbr of listeners currently supported */ + sbool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ + sbool bUsingEPoll; /**< are we in epoll mode (means we do not need to keep track of sessions!) */ + int iLstnCurr; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ + int iLstnMax; /**< max number of listners supported */ int iSessMax; /**< max number of sessions supported */ tcpLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ + int addtlFrameDelim; /**< additional frame delimiter for plain TCP syslog framing (e.g. to handle NetScreen) */ + int bDisableLFDelim; /**< if 1, standard LF frame delimiter is disabled (*very dangerous*) */ tcps_sess_t **pSessions;/**< array of all of our sessions */ void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ /* callbacks */ @@ -107,8 +115,14 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetSessMax)(tcpsrv_t *pThis, int iMaxSess); /* 2009-04-09 */ /* added v6 */ rsRetVal (*SetOnMsgReceive)(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); /* 2009-05-24 */ + rsRetVal (*SetRuleset)(tcpsrv_t *pThis, ruleset_t*); /* 2009-06-12 */ + /* added v7 (accidently named v8!) */ + rsRetVal (*SetLstnMax)(tcpsrv_t *pThis, int iMaxLstn); /* 2009-08-17 */ + rsRetVal (*SetNotificationOnRemoteClose)(tcpsrv_t *pThis, int bNewVal); /* 2009-10-01 */ + /* added v9 -- rgerhards, 2010-03-01 */ + rsRetVal (*SetbDisableLFDelim)(tcpsrv_t*, int); ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 @@ -49,62 +49,63 @@ static struct template *tplLastStatic = NULL; /* last static element of the temp -/* This functions converts a template into a string. It should - * actually be in template.c, but this requires larger re-structuring - * of the code (because all the property-access functions are static - * to this module). I have placed it next to the iov*() functions, as - * it is somewhat similiar in what it does. - * - * The function takes a pointer to a template and a pointer to a msg object. - * It the creates a string based on the template definition. A pointer - * to that string is returned to the caller. The caller MUST FREE that - * pointer when it is no longer needed. If the function fails, NULL - * is returned. - * If memory allocation fails in this function, we silently return - * NULL. The reason is that we can not do anything against it. And - * if we raise an alert, the memory situation might become even - * worse. So we prefer to let the caller deal with it. - * rgerhards, 2007-07-03 +/* helper to tplToString, extends buffer */ +#define ALLOC_INC 128 +static inline rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize) +{ + uchar *pNewBuf; + size_t iNewSize; + DEFiRet; + + iNewSize = (iMinSize / ALLOC_INC + 1) * ALLOC_INC; + CHKmalloc(pNewBuf = (uchar*) realloc(*pBuf, iNewSize)); + *pBuf = pNewBuf; + *pLenBuf = iNewSize; +dbgprintf("extend buf to at least %ld, done %ld\n", iMinSize, iNewSize); + +finalize_it: + RETiRet; +} + + +/* This functions converts a template into a string. * - * rgerhards, 2007-09-05: I changed the interface to use the standard iRet - * "calling sequence". This greatly eases complexity when it comes to handling - * errors in called modules (plus, it is much nicer). + * The function takes a pointer to a template and a pointer to a msg object + * as well as a pointer to an output buffer and its size. Note that the output + * buffer pointer may be NULL, size 0, in which case a new one is allocated. + * The outpub buffer is grown as required. It is the caller's duty to free the + * buffer when it is done. Note that it is advisable to reuse memory, as this + * offers big performance improvements. + * rewritten 2009-06-19 rgerhards */ -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) +rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *pLenBuf) { DEFiRet; struct templateEntry *pTpe; - cstr_t *pCStr; + int iBuf; unsigned short bMustBeFreed; uchar *pVal; size_t iLenVal; assert(pTpl != NULL); assert(pMsg != NULL); - assert(ppSz != NULL); + assert(ppBuf != NULL); + assert(pLenBuf != NULL); /* loop through the template. We obtain one value * and copy it over to our dynamic string buffer. Then, we * free the obtained value (if requested). We continue this * loop until we got hold of all values. */ - CHKiRet(rsCStrConstruct(&pCStr)); - pTpe = pTpl->pEntryRoot; + iBuf = 0; while(pTpe != NULL) { if(pTpe->eEntryType == CONSTANT) { - CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, - (uchar *) pTpe->data.constant.pConstant, - pTpe->data.constant.iLenConstant) - ) { - dbgprintf("error %d during tplToString()\n", iRet); - /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); - FINALIZE; - } + pVal = (uchar*) pTpe->data.constant.pConstant; + iLenVal = pTpe->data.constant.iLenConstant; + bMustBeFreed = 0; } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); - iLenVal = strlen((char*) pVal); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, &iLenVal, &bMustBeFreed); /* we now need to check if we should use SQL option. In this case, * we must go over the generated string and escape '\'' characters. * rgerhards, 2005-09-22: the option values below look somewhat misplaced, @@ -115,30 +116,25 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); else if(pTpl->optFormatForSQL == 2) doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); - /* value extracted, so lets copy */ - CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, (uchar*) pVal, iLenVal)) { - dbgprintf("error %d during tplToString()\n", iRet); - /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); - if(bMustBeFreed) - free(pVal); - FINALIZE; - } - if(bMustBeFreed) - free(pVal); } + /* got source, now copy over */ + if(iBuf + iLenVal >= *pLenBuf) /* we reserve one char for the final \0! */ + CHKiRet(ExtendBuf(ppBuf, pLenBuf, iBuf + iLenVal + 1)); + + if(iLenVal > 0) { /* may be zero depending on property */ + memcpy(*ppBuf + iBuf, pVal, iLenVal); + iBuf += iLenVal; + } + + if(bMustBeFreed) + free(pVal); + pTpe = pTpe->pNext; } - /* we are done with the template, now let's convert the result into a - * "real" (usable) string and discard the helper structures. - */ - CHKiRet(rsCStrFinish(pCStr)); - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pVal, 0)); + (*ppBuf)[iBuf] = '\0'; /* space was reserved above (see copy) */ finalize_it: - *ppSz = (iRet == RS_RET_OK) ? pVal : NULL; - RETiRet; } @@ -158,6 +154,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) struct templateEntry *pTpe; uchar **pArr; int iArr; + size_t propLen; unsigned short bMustBeFreed; uchar *pVal; @@ -178,7 +175,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) if(pTpe->eEntryType == CONSTANT) { CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, &propLen, &bMustBeFreed); if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ pArr[iArr] = pVal; /* ... so we can use it! */ } else { @@ -272,21 +269,21 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM p = *pp; iLen = *pLen; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); while(*p) { if(*p == '\'') { - CHKiRet(rsCStrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); iLen++; /* reflect the extra character */ } else if((escapeMode == 1) && (*p == '\\')) { - CHKiRet(rsCStrAppendChar(pStrB, '\\')); + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); if(*pbMustBeFreed) free(*pp); /* discard previous value */ @@ -299,7 +296,7 @@ finalize_it: if(iRet != RS_RET_OK) { doSQLEmergencyEscape(*pp, escapeMode); if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -376,9 +373,8 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) p = *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; - rsCStrSetAllocIncrement(pStrB, 32); /* process the message and expand escapes * (additional escapes can be added here if needed) */ @@ -387,22 +383,22 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) switch(*++p) { case '\0': /* the best we can do - it's invalid anyhow... */ - rsCStrAppendChar(pStrB, *p); + cstrAppendChar(pStrB, *p); break; case 'n': - rsCStrAppendChar(pStrB, '\n'); + cstrAppendChar(pStrB, '\n'); ++p; break; case 'r': - rsCStrAppendChar(pStrB, '\r'); + cstrAppendChar(pStrB, '\r'); ++p; break; case '\\': - rsCStrAppendChar(pStrB, '\\'); + cstrAppendChar(pStrB, '\\'); ++p; break; case '%': - rsCStrAppendChar(pStrB, '%'); + cstrAppendChar(pStrB, '%'); ++p; break; case '0': /* numerical escape sequence */ @@ -419,33 +415,30 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) while(*p && isdigit((int)*p)) { i = i * 10 + *p++ - '0'; } - rsCStrAppendChar(pStrB, i); + cstrAppendChar(pStrB, i); break; default: - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); break; } } else - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); } if((pTpe = tpeConstruct(pTpl)) == NULL) { - /* OK, we are out of luck. Let's invalidate the - * entry and that's it. - */ - pTpe->eEntryType = UNDEFINED; + rsCStrDestruct(&pStrB); return 1; } pTpe->eEntryType = CONSTANT; - rsCStrFinish(pStrB); + cstrFinalize(pStrB); /* We obtain the length from the counted string object * (before we delete it). Later we might take additional * benefit from the counted string object. * 2005-09-09 rgerhards */ pTpe->data.constant.iLenConstant = rsCStrLen(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) + if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) return 1; *pp = p; @@ -552,7 +545,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) p = (unsigned char*) *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -563,14 +556,18 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) pTpe->eEntryType = FIELD; while(*p && *p != '%' && *p != ':') { - rsCStrAppendChar(pStrB, tolower(*p)); + cstrAppendChar(pStrB, tolower(*p)); ++p; /* do NOT do this in tolower()! */ } - /* got the name*/ - rsCStrFinish(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) + /* got the name */ + cstrFinalize(pStrB); + + if(propNameToID(pStrB, &pTpe->data.field.propid) != RS_RET_OK) { + cstrDestruct(&pStrB); return 1; + } + cstrDestruct(&pStrB); /* no longer needed, now use ID */ /* Check frompos, if it has an R, then topos should be a regex */ if(*p == ':') { @@ -743,7 +740,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) /* We get here ONLY if the regex end was found */ longitud = regex_end - p; /* Malloc for the regex string */ - regex_char = (unsigned char *) malloc(longitud + 1); + regex_char = (unsigned char *) MALLOC(longitud + 1); if(regex_char == NULL) { dbgprintf("Could not allocate memory for template parameter!\n"); pTpe->data.field.has_regex = 0; @@ -849,7 +846,7 @@ struct template *tplAddLine(char* pName, unsigned char** ppRestOfConfLine) return NULL; pTpl->iLenName = strlen(pName); - pTpl->pszName = (char*) malloc(sizeof(char) * (pTpl->iLenName + 1)); + pTpl->pszName = (char*) MALLOC(sizeof(char) * (pTpl->iLenName + 1)); if(pTpl->pszName == NULL) { dbgprintf("tplAddLine could not alloc memory for template name!"); pTpl->iLenName = 0; @@ -1009,8 +1006,6 @@ void tplDeleteAll(void) regexp.regfree(&(pTpeDel->data.field.re)); } } - /*dbgprintf("(FIELD), value: '%s'", pTpeDel->data.field.pPropRepl);*/ - free(pTpeDel->data.field.pPropRepl); break; } /*dbgprintf("\n");*/ @@ -1066,8 +1061,6 @@ void tplDeleteNew(void) regexp.regfree(&(pTpeDel->data.field.re)); } } - /*dbgprintf("(FIELD), value: '%s'", pTpeDel->data.field.pPropRepl);*/ - free(pTpeDel->data.field.pPropRepl); break; } /*dbgprintf("\n");*/ @@ -1116,7 +1109,7 @@ void tplPrintList(void) pTpe->data.constant.pConstant); break; case FIELD: - dbgprintf("(FIELD), value: '%s' ", pTpe->data.field.pPropRepl); + dbgprintf("(FIELD), value: '%d' ", pTpe->data.field.propid); switch(pTpe->data.field.eDateFormat) { case tplFmtDefault: break; @@ -63,7 +63,7 @@ struct templateEntry { int iLenConstant; /* its length */ } constant; struct { - uchar *pPropRepl; /* pointer to property replacer string */ + propid_t propid; /* property to be used */ unsigned iFromPos; /* for partial strings only chars from this position ... */ unsigned iToPos; /* up to that one... */ #ifdef FEATURE_REGEXP @@ -127,7 +127,7 @@ void tplLastStaticInit(struct template *tpl); * rgerhards, 2007-08-06 */ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz); +rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); diff --git a/tests/DiagTalker.java b/tests/DiagTalker.java index e33a5867..5a6f7dd5 100644 --- a/tests/DiagTalker.java +++ b/tests/DiagTalker.java @@ -1,3 +1,24 @@ +/* A yet very simple tool to talk to imdiag. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ //package com.rsyslog.diag; import java.io.*; import java.net.*; @@ -9,10 +30,14 @@ public class DiagTalker { PrintWriter out = null; BufferedReader in = null; final String host = "127.0.0.1"; - final int port = 13500; + int port = 13500; + if(args.length > 1) { + port = Integer.parseInt(args[1]); + } try { diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(0); /* wait for lenghty operations */ out = new PrintWriter(diagSocket.getOutputStream(), true); in = new BufferedReader(new InputStreamReader( diagSocket.getInputStream())); @@ -29,9 +54,14 @@ public class DiagTalker { new InputStreamReader(System.in)); String userInput; - while ((userInput = stdIn.readLine()) != null) { - out.println(userInput); - System.out.println("imdiag returns: " + in.readLine()); + try { + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + } catch (SocketException e) { + System.err.println("We had a socket exception and consider this to be OK: " + + e.getMessage()); } out.close(); diff --git a/tests/Makefile.am b/tests/Makefile.am index caa95c51..63dba939 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,17 +1,67 @@ +if ENABLE_TESTBENCH TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh +TESTS = $(TESTRUNS) cfg.sh \ + arrayqueue.sh \ + linkedlistqueue.sh \ + da-mainmsg-q.sh \ + validation-run.sh \ + imtcp-multiport.sh \ + daqueue-persist.sh \ + diskqueue.sh \ + diskqueue-fsync.sh \ + rulesetmultiqueue.sh \ + manytcp.sh \ + rsf_getenv.sh \ + sndrcv.sh \ + sndrcv_gzip.sh \ + sndrcv_udp.sh \ + sndrcv_udp_nonstdpt.sh \ + queue-persist.sh \ + pipeaction.sh \ + execonlyonce.sh \ + dircreate_dflt.sh \ + dircreate_off.sh \ + queue-persist.sh + +if ENABLE_OMUDPSPOOF +TESTS += sndrcv_omudpspoof.sh \ + sndrcv_omudpspoof_nonstdpt.sh +endif + if ENABLE_OMSTDOUT -TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh +TESTS += omod-if-array.sh \ + proprepltest.sh \ + parsertest.sh \ + timestamp.sh \ + inputname.sh \ + threadingmq.sh \ + threadingmqaq.sh \ + discard.sh \ + badqi.sh \ + tabescape_dflt.sh \ + tabescape_off.sh \ + fieldtest.sh +endif + +if ENABLE_OMRULESET +TESTS += omruleset.sh \ + omruleset-queue.sh endif + +check_JAVA = DiagTalker.java + +endif # if ENABLE_TESTBENCH + TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class test_files = testbench.h runtime-dummy.c -check_JAVA = DiagTalker.java -#dist_java_JAVA = DiagTalker.java - EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ + validation-run.sh \ + testsuites/invalid.conf \ + testsuites/valid.conf \ + cfg.sh \ cfg1.cfgtest \ cfg1.testin \ cfg2.cfgtest \ @@ -23,33 +73,151 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ + timestamp.sh \ + testsuites/ts3164.conf \ + testsuites/mon1digit.ts3164 \ + testsuites/mon2digit.ts3164 \ + testsuites/Jan.ts3164 \ + testsuites/Feb.ts3164 \ + testsuites/Mar.ts3164 \ + testsuites/Apr.ts3164 \ + testsuites/May.ts3164 \ + testsuites/Jun.ts3164 \ + testsuites/Jul.ts3164 \ + testsuites/Aug.ts3164 \ + testsuites/Sep.ts3164 \ + testsuites/Oct.ts3164 \ + testsuites/Nov.ts3164 \ + testsuites/Dec.ts3164 \ + testsuites/ts3339.conf \ + testsuites/master.ts3339 \ + testsuites/tsmysql.conf \ + testsuites/master.tsmysql \ + testsuites/tspgsql.conf \ + testsuites/master.tspgsql \ + testsuites/subsecond.conf \ + testsuites/master.subsecond \ + testsuites/parse_8bit_escape.conf \ + testsuites/8bit.parse_8bit_escape \ testsuites/parse1.conf \ + testsuites/field1.conf \ testsuites/1.parse1 \ - testsuites/rfc3164.parse1 \ + testsuites/2.parse1 \ + testsuites/3.parse1 \ + testsuites/4.parse1 \ + testsuites/mark.parse1 \ + testsuites/8bit.parse1 \ + testsuites/empty.parse1 \ + testsuites/oversizeTag-1.parse1 \ + testsuites/weird.parse1 \ + testsuites/date1.parse1 \ + testsuites/date2.parse1 \ + testsuites/date3.parse1 \ + testsuites/date4.parse1 \ + testsuites/date5.parse1 \ + testsuites/rfc3164.parse1 \ testsuites/rfc5424-1.parse1 \ testsuites/rfc5424-2.parse1 \ testsuites/rfc5424-3.parse1 \ testsuites/rfc5424-4.parse1 \ + testsuites/malformed1.parse1 \ + testsuites/reallife.parse1 \ + testsuites/parse2.conf \ + testsuites/reallife.parse2 \ + testsuites/parse3.conf \ + testsuites/reallife.parse3 \ + testsuites/parse_invld_regex.conf \ + testsuites/samples.parse_invld_regex \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ + testsuites/1.field1 \ killrsyslog.sh \ parsertest.sh \ + fieldtest.sh \ + rsf_getenv.sh \ + testsuites/rsf_getenv.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ + arrayqueue.sh \ + testsuites/arrayqueue.conf \ + linkedlistqueue.sh \ + testsuites/linkedlistqueue.conf \ + da-mainmsg-q.sh \ + testsuites/da-mainmsg-q.conf \ + diskqueue-fsync.sh \ + testsuites/diskqueue-fsync.conf \ imtcp-multiport.sh \ testsuites/imtcp-multiport.conf \ manytcp.sh \ testsuites/manytcp.conf \ - fieldtest.sh \ - testsuites/field1.conf \ - testsuites/1.field1 \ inputname.sh \ testsuites/inputname_imtcp.conf \ testsuites/1.inputname_imtcp_12514 \ testsuites/1.inputname_imtcp_12515 \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ - waitqueueempty.sh \ + discard.sh \ + testsuites/discard.conf \ + diag.sh \ + testsuites/diag-common.conf \ + testsuites/diag-common2.conf \ + daqueue-persist.sh \ + daqueue-persist-drvr.sh \ + queue-persist.sh \ + queue-persist-drvr.sh \ + testsuites/queue-persist.conf \ + threadingmq.sh \ + testsuites/threadingmq.conf \ + threadingmqaq.sh \ + testsuites/threadingmqaq.conf \ + sndrcv_drvr.sh \ + sndrcv.sh \ + testsuites/sndrcv_sender.conf \ + testsuites/sndrcv_rcvr.conf \ + sndrcv_udp.sh \ + testsuites/sndrcv_udp_sender.conf \ + testsuites/sndrcv_udp_rcvr.conf \ + sndrcv_udp_nonstdpt.sh \ + testsuites/sndrcv_udp_nonstdpt_sender.conf \ + testsuites/sndrcv_udp_nonstdpt_rcvr.conf \ + sndrcv_omudpspoof.sh \ + testsuites/sndrcv_omudpspoof_sender.conf \ + testsuites/sndrcv_omudpspoof_rcvr.conf \ + sndrcv_omudpspoof_nonstdpt.sh \ + testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf \ + testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf \ + sndrcv_gzip.sh \ + testsuites/sndrcv_gzip_sender.conf \ + testsuites/sndrcv_gzip_rcvr.conf \ + pipeaction.sh \ + testsuites/pipeaction.conf \ + proprepltest.sh \ + testsuites/rfctag.conf \ + testsuites/master.rfctag \ + testsuites/nolimittag.conf \ + testsuites/master.nolimittag \ + rulesetmultiqueue.sh \ + testsuites/rulesetmultiqueue.conf \ + omruleset.sh \ + testsuites/omruleset.conf \ + omruleset-queue.sh \ + testsuites/omruleset-queue.conf \ + badqi.sh \ + testsuites/badqi.conf \ + bad_qi/dbq.qi \ + execonlyonce.sh \ + testsuites/execonlyonce.conf \ + testsuites/execonlyonce.data \ + tabescape_dflt.sh \ + testsuites/tabescape_dflt.conf \ + testsuites/1.tabescape_dflt \ + tabescape_off.sh \ + testsuites/tabescape_off.conf \ + testsuites/1.tabescape_off \ + dircreate_dflt.sh \ + testsuites/dircreate_dflt.conf \ + dircreate_off.sh \ + testsuites/dircreate_off.conf \ DiagTalker.java \ cfg.sh diff --git a/tests/arrayqueue.sh b/tests/arrayqueue.sh new file mode 100755 index 00000000..71e1cc21 --- /dev/null +++ b/tests/arrayqueue.sh @@ -0,0 +1,18 @@ +# Test for fixedArray queue mode +# added 2009-05-20 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[arrayqueue.sh\]: testing queue fixedArray queue mode +source $srcdir/diag.sh init +source $srcdir/diag.sh startup arrayqueue.conf + +# 40000 messages should be enough +source $srcdir/diag.sh injectmsg 0 40000 + +# terminate *now* (don't wait for queue to drain!) +kill `cat rsyslog.pid` + +# now wait until rsyslog.pid is gone (and the process finished) +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/bad_qi/dbq.qi b/tests/bad_qi/dbq.qi new file mode 100644 index 00000000..5f56e41a --- /dev/null +++ b/tests/bad_qi/dbq.qi @@ -0,0 +1,29 @@ +!OPB:1:queue:1: ++iQueueSize:2:2:84: ++iUngottenObjs:2:1:1: ++tVars.disk.sizeOnDisk:2:5:57906: ++tVars.disk.bytesRead:2:4:1010: +>End +. +<Obj:1:strm:1: ++iCurrFNum:2:1:1: ++pszFName:1:3:dbq: ++iMaxFiles:2:8:10000000: ++bDeleteOnClose:2:1:0: ++sType:2:1:1: ++tOperationsMode:2:1:2: ++tOpenMode:2:3:384: ++iCurrOffs:2:5:57906: +>End +. +<Obj:1:strm:1: ++iCurrFNum:2:1:1: ++pszFName:1:3:dbq: ++iMaxFiles:2:8:10000000: ++bDeleteOnClose:2:1:1: ++sType:2:1:1: ++tOperationsMode:2:1:1: ++tOpenMode:2:3:384: ++iCurrOffs:2:4:1010: +>End +. diff --git a/tests/badqi.sh b/tests/badqi.sh new file mode 100755 index 00000000..81f8c1c2 --- /dev/null +++ b/tests/badqi.sh @@ -0,0 +1,16 @@ +# Test for a startup with a bad qi file. This tests simply tests +# if the rsyslog engine survives (we had segfaults in this situation +# in the past). +# added 2009-10-21 by RGerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =============================================================================== +echo \[badqi.sh\]: test startup with invalid .qi file +source $srcdir/diag.sh init +source $srcdir/diag.sh startup badqi.conf +# we just inject a handful of messages so that we have something to wait for... +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # wait for process to terminate +source $srcdir/diag.sh seq-check 0 19 +source $srcdir/diag.sh exit diff --git a/tests/cfg.sh b/tests/cfg.sh index cb838354..f850c4d1 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -31,6 +31,7 @@ # # A copy of the GPL can be found in the file "COPYING" in this distribution. #set -x +echo \[cfg.sh\]: rm -f tmp echo "local directory" # diff --git a/tests/chkseq.c b/tests/chkseq.c index 3203c250..6334d787 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -3,9 +3,10 @@ * be set. * * Params - * argv[1] file to check - * argv[2] start number - * argv[3] end number + * -f<filename> MUST be given! + * -s<starting number> -e<ending number> + * default for s is 0. -e should be given (else it is also 0) + * -d may be specified, in which case duplicate messages are permitted. * * Part of the testbench for rsyslog. * @@ -31,6 +32,7 @@ #include "config.h" #include <stdio.h> #include <stdlib.h> +#include <getopt.h> int main(int argc, char *argv[]) { @@ -38,39 +40,81 @@ int main(int argc, char *argv[]) int val; int i; int ret = 0; - int start, end; + int verbose = 0; + int dupsPermitted = 0; + int start = 0, end = 0; + int opt; + int nDups = 0; + char *file = NULL; - if(argc != 4) { - printf("Invalid call of chkseq\n"); - printf("Usage: chkseq file start end\n"); + while((opt = getopt(argc, argv, "e:f:ds:v")) != EOF) { + switch((char)opt) { + case 'f': + file = optarg; + break; + case 'd': + dupsPermitted = 1; + break; + case 'e': + end = atoi(optarg); + break; + case 's': + start = atoi(optarg); + break; + case 'v': + ++verbose; + break; + default:printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file -sstart -eend -d\n"); + exit(1); + } + } + + if(file == NULL) { + printf("file must be given!\n"); exit(1); } - - start = atoi(argv[2]); - end = atoi(argv[3]); if(start > end) { printf("start must be less than or equal end!\n"); exit(1); } + if(verbose) { + printf("chkseq: start %d, end %d\n", start, end); + } + /* read file */ - fp = fopen(argv[1], "r"); + fp = fopen(file, "r"); if(fp == NULL) { - perror(argv[1]); + printf("error opening file '%s'\n", file); + perror(file); exit(1); } - for(i = start ; i < end ; ++i) { + for(i = start ; i < end+1 ; ++i) { if(fscanf(fp, "%d\n", &val) != 1) { printf("scanf error in index i=%d\n", i); exit(1); } if(val != i) { - printf("read value %d, but expected value %d\n", val, i); - exit(1); + if(val == i - 1 && dupsPermitted) { + --i; + ++nDups; + } else { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } } } + if(nDups != 0) + printf("info: had %d duplicates (this is no error)\n", nDups); + + if(i - 1 != end) { + printf("only %d records in file, expected %d\n", i - 1, end); + exit(1); + } + exit(ret); } diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh new file mode 100755 index 00000000..d9cc0d4d --- /dev/null +++ b/tests/da-mainmsg-q.sh @@ -0,0 +1,33 @@ +# Test for DA mode on the main message queue +# This test checks if DA mode operates correctly. To do so, +# it uses a small in-memory queue size, so that DA mode is initiated +# rather soon, and disk spooling used. There is some uncertainty (based +# on machine speeds), but in general the test should work rather well. +# We add a few messages after the initial run, just so that we can +# check everything recovers from DA mode correctly. +# added 2009-04-22 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo "[da-mainmsg-q.sh]: testing main message queue in DA mode (going to disk)" +source $srcdir/diag.sh init +source $srcdir/diag.sh startup da-mainmsg-q.conf + +# part1: send first 50 messages (in memory, only) +#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 50 +source $srcdir/diag.sh injectmsg 0 50 +source $srcdir/diag.sh wait-queueempty # let queue drain for this test case + +# part 2: send bunch of messages. This should trigger DA mode +#source $srcdir/diag.sh injectmsg 50 20000 +source $srcdir/diag.sh injectmsg 50 2000 +ls -l test-spool # for manual review + +# send another handful +source $srcdir/diag.sh injectmsg 2050 50 +#sleep 1 # we need this so that rsyslogd can receive all outstanding messages + +# clean up and check test result +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 2099 +source $srcdir/diag.sh exit diff --git a/tests/daqueue-persist-drvr.sh b/tests/daqueue-persist-drvr.sh new file mode 100755 index 00000000..7934eb2b --- /dev/null +++ b/tests/daqueue-persist-drvr.sh @@ -0,0 +1,36 @@ +# Test for queue data persisting at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo \[daqueue-persist-drvr.sh\]: testing memory daqueue persisting to disk, mode $1 +source $srcdir/diag.sh init + +#export RSYSLOG_DEBUG="debug nologfuncflow nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="log" + +# prepare config +echo \$MainMsgQueueType $1 > work-queuemode.conf +echo "*.* :omtesting:sleep 0 1000" > work-delay.conf + +# inject 10000 msgs, so that DO hit the high watermark +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh injectmsg 0 10000 +$srcdir/diag.sh shutdown-immediate +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh check-mainq-spool + +echo "Enter phase 2, rsyslogd restart" + +exit + +# restart engine and have rest processed +#remove delay +echo "#" > work-delay.conf +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99999 +source $srcdir/diag.sh exit diff --git a/tests/daqueue-persist.sh b/tests/daqueue-persist.sh new file mode 100755 index 00000000..feb2a347 --- /dev/null +++ b/tests/daqueue-persist.sh @@ -0,0 +1,12 @@ +# Test for queue data persisting at shutdown. We use the actual driver +# to carry out multiple tests with different queue modes +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo \[daqueue-persist.sh\]: test data persisting at shutdown +source $srcdir/daqueue-persist-drvr.sh LinkedList +source $srcdir/daqueue-persist-drvr.sh FixedArray +# the disk test should not fail, however, the config is extreme and using +# it more or less is a config error +source $srcdir/daqueue-persist-drvr.sh Disk +# we do not test Direct mode because this absolute can not work in direct mode +# (maybe we should do a fail-type of test?) diff --git a/tests/diag.sh b/tests/diag.sh new file mode 100755 index 00000000..1f410645 --- /dev/null +++ b/tests/diag.sh @@ -0,0 +1,124 @@ +# this shell script provides commands to the common diag system. It enables +# test scripts to wait for certain conditions and initiate certain actions. +# needs support in config file. +# NOTE: this file should be included with "source diag.sh", as it otherwise is +# not always able to convey back states to the upper-level test driver +# begun 2009-05-27 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +#valgrind="valgrind --malloc-fill=ff --free-fill=fe --log-fd=1" +#valgrind="valgrind --tool=drd --log-fd=1" +#valgrind="valgrind --tool=helgrind --log-fd=1" +#set -o xtrace +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="log" +case $1 in + 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + cp $srcdir/testsuites/diag-common.conf diag-common.conf + cp $srcdir/testsuites/diag-common2.conf diag-common2.conf + rm -f rsyslogd.started work-*.conf + rm -f rsyslogd2.started work-*.conf + rm -f work rsyslog.out.log rsyslog.out.log.save # common work files + rm -rf test-spool test-logdir + rm -f core.* vgcore.* + mkdir test-spool + ;; + 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf + rm -f rsyslogd2.started diag-common2.conf + rm -f work rsyslog.out.log rsyslog.out.log.save # common work files + rm -rf test-spool test-logdir + echo ------------------------------------------------------------------------------- + ;; + 'startup') # start rsyslogd with default params. $2 is the config file name to use + # returns only after successful startup, $3 is the instance (blank or 2!) + $valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $srcdir/diag.sh wait-startup $3 + ;; + 'wait-startup') # wait for rsyslogd startup ($2 is the instance) + while test ! -f rsyslogd$2.started; do + true + done + echo "rsyslogd$2 started with pid " `cat rsyslog$2.pid` + ;; + 'wait-shutdown') # actually, we wait for rsyslog.pid to be deleted. $2 is the + # instance + while test -f rsyslog$2.pid; do + true + done + if [ -e core.* ] + then + echo "ABORT! core file exists, starting interactive shell" + bash + exit 1 + fi + ;; + 'wait-queueempty') # wait for main message queue to be empty. $2 is the instance. + if [ "$2" == "2" ] + then + echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker + else + echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker 13501 + fi + ;; + 'shutdown-when-empty') # shut rsyslogd down when main queue is empty. $2 is the instance. + $srcdir/diag.sh wait-queueempty $2 + kill `cat rsyslog$2.pid` + # note: we do not wait for the actual termination! + ;; + 'shutdown-immediate') # shut rsyslogd down without emptying the queue. $2 is the instance. + kill `cat rsyslog.pid` + # note: we do not wait for the actual termination! + ;; + 'tcpflood') # do a tcpflood run and check if it worked params are passed to tcpflood + ./tcpflood $2 $3 $4 $5 $6 $7 $8 + if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save + exit 1 + fi + ;; + 'injectmsg') # inject messages via our inject interface (imdiag) + echo injecting $3 messages + echo injectmsg $2 $3 $4 $5 | java -classpath $abs_top_builddir DiagTalker + # TODO: some return state checking? (does it really make sense here?) + ;; + 'check-mainq-spool') # check if mainqueue spool files exist, if not abort (we just check .qi). + echo There must exist some files now: + ls -l test-spool + if test ! -f test-spool/mainq.qi; then + echo "error: mainq.qi does not exist where expected to do so!" + ls -l test-spool + exit 1 + fi + ;; + 'seq-check') # do the usual sequence check to see if everything was properly received. $2 is the instance. + rm -f work + sort < rsyslog.out.log > work + # $4... are just to have the abilit to pass in more options... + # add -v to chkseq if you need more verbose output + ./chkseq -fwork -s$2 -e$3 $4 $5 $6 $7 + if [ "$?" -ne "0" ]; then + echo "sequence error detected" + exit 1 + fi + ;; + 'seq-check2') # do the usual sequence check to see if everything was properly received. This is + # a duplicateof seq-check, but we could not change its calling conventions without + # breaking a lot of exitings test cases, so we preferred to duplicate the code here. + rm -f work2 + sort < rsyslog2.out.log > work2 + # $4... are just to have the abilit to pass in more options... + ./chkseq -fwork2 -v -s$2 -e$3 $4 $5 $6 $7 + if [ "$?" -ne "0" ]; then + echo "sequence error detected" + exit 1 + fi + ;; + 'nettester') # perform nettester-based tests + # use -v for verbose output! + ./nettester -t$2 -i$3 $4 + if [ "$?" -ne "0" ]; then + exit 1 + fi + ;; + *) echo "invalid argument" $1 +esac diff --git a/tests/dircreate_dflt.sh b/tests/dircreate_dflt.sh new file mode 100755 index 00000000..71a671f3 --- /dev/null +++ b/tests/dircreate_dflt.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_dflt_dflt.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_dflt.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ ! -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile not created!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/dircreate_off.sh b/tests/dircreate_off.sh new file mode 100755 index 00000000..92fdee01 --- /dev/null +++ b/tests/dircreate_off.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_off_off.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_off.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile WAS created where not permitted to!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/discard.sh b/tests/discard.sh new file mode 100755 index 00000000..9824b361 --- /dev/null +++ b/tests/discard.sh @@ -0,0 +1,17 @@ +# Test for discard functionality +# This test checks if discard works. It is not a perfect test but +# will find at least segfaults and obviously not discarded messages. +# added 2009-07-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =============================================================================== +echo \[discard.sh\]: testing discard functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup discard.conf +# 20000 messages should be enough - the disk test is slow enough ;) +sleep 4 +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10 1 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 2 10 +source $srcdir/diag.sh exit diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh new file mode 100755 index 00000000..c826d32c --- /dev/null +++ b/tests/diskqueue-fsync.sh @@ -0,0 +1,16 @@ +# Test for disk-only queue mode (with fsync for queue files) +# This test checks if queue files can be correctly written +# and read back, but it does not test the transition from +# memory to disk mode for DA queues. +# added 2009-06-09 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo \[diskqueue-fsync.sh\]: testing queue disk-only mode, fsync case +source $srcdir/diag.sh init +source $srcdir/diag.sh startup diskqueue-fsync.conf +# 1000 messages should be enough - the disk fsync test is very slow! +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 1000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 999 +source $srcdir/diag.sh exit diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index eabfcf78..e91f0633 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -5,31 +5,13 @@ # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -#set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="tmp" -echo testing queue disk-only mode -rm -rf test-spool -mkdir test-spool -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +echo \[diskqueue.sh\]: testing queue disk-only mode +source $srcdir/diag.sh init +source $srcdir/diag.sh startup diskqueue.conf # 20000 messages should be enough - the disk test is slow enough ;) -./tcpflood 127.0.0.1 13514 1 20000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 19999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log -rm -rf test-spool +sleep 4 +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 19999 +source $srcdir/diag.sh exit diff --git a/tests/execonlyonce.sh b/tests/execonlyonce.sh new file mode 100755 index 00000000..6ddc89b5 --- /dev/null +++ b/tests/execonlyonce.sh @@ -0,0 +1,29 @@ +# Test for the $ActionExecOnlyOnceEveryInterval directive. +# We inject a couple of messages quickly during the interval, +# then wait until the interval expires, then quickly inject +# another set. After that, it is checked if exactly two messages +# have arrived. +# The once interval must be set to 3 seconds in the config file. +# added 2009-11-12 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[execonlyonce.sh\]: test for the $ActionExecOnlyOnceEveryInterval directive +source $srcdir/diag.sh init +source $srcdir/diag.sh startup execonlyonce.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10 1 +# now wait until the interval definitely expires +sleep 4 # one more than the once inerval! +# and inject another couple of messages +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10 100 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown + +# now we need your custom logic to see if the result is equal to the +# expected result +cmp rsyslog.out.log testsuites/execonlyonce.data +if [ $? -eq 1 ] +then + echo "ERROR, output not as expected" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/fieldtest.sh b/tests/fieldtest.sh index 482fa143..9875fda6 100755 --- a/tests/fieldtest.sh +++ b/tests/fieldtest.sh @@ -1,4 +1,4 @@ -echo test fieldtest via udp +echo \[fieldtest.sh\]: test fieldtest via udp $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tfield1 -iudp diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh index 17480dae..47a33cb9 100755 --- a/tests/imtcp-multiport.sh +++ b/tests/imtcp-multiport.sh @@ -7,74 +7,36 @@ # # added 2009-05-22 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 -echo testing imtcp multiple listeners -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13514 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +echo =============================================================================== +echo \[imtcp-multiport.sh\]: testing imtcp multiple listeners +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit # # # ### now complete new cycle with other port ### # # -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13515 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13515 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit # # # ### now complete new cycle with other port ### # # -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13516 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13516 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit diff --git a/tests/inputname.sh b/tests/inputname.sh index e1a58517..71f11c1e 100755 --- a/tests/inputname.sh +++ b/tests/inputname.sh @@ -1,4 +1,4 @@ -echo testing $InputTCPServerInputName directive +echo \[inputname.sh\]: testing $InputTCPServerInputName directive $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason echo port 12514 diff --git a/tests/killrsyslog.sh b/tests/killrsyslog.sh index b1be757b..aac24909 100755 --- a/tests/killrsyslog.sh +++ b/tests/killrsyslog.sh @@ -2,6 +2,12 @@ if [ -e "rsyslog.pid" ] then echo rsyslog.pid exists, trying to shut down rsyslogd process `cat rsyslog.pid`. - kill `cat rsyslog.pid` + kill -9 `cat rsyslog.pid` + sleep 1 +fi +if [ -e "rsyslog2.pid" ] +then + echo rsyslog2.pid exists, trying to shut down rsyslogd process `cat rsyslog2.pid`. + kill -9 `cat rsyslog2.pid` sleep 1 fi diff --git a/tests/linkedlistqueue.sh b/tests/linkedlistqueue.sh new file mode 100755 index 00000000..e6d48a68 --- /dev/null +++ b/tests/linkedlistqueue.sh @@ -0,0 +1,17 @@ +# Test for Linkedlist queue mode +# added 2009-05-20 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo \[linkedlistqueue.sh\]: testing queue Linkedlist queue mode +source $srcdir/diag.sh init +source $srcdir/diag.sh startup linkedlistqueue.conf + +# 40000 messages should be enough +source $srcdir/diag.sh injectmsg 0 40000 + +# terminate *now* (don't wait for queue to drain) +kill `cat rsyslog.pid` + +# now wait until rsyslog.pid is gone (and the process finished) +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 06bd38b6..06e2fac9 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,21 +1,9 @@ -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +# test many concurrent tcp connections +echo \[manytcp.sh\]: test concurrent tcp connections +source $srcdir/diag.sh init +source $srcdir/diag.sh startup manytcp.conf # the config file specifies exactly 1100 connections -./tcpflood 127.0.0.1 13514 1000 40000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 39999 -if [ "$?" -ne "0" ]; then - rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1000 40000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/nettester.c b/tests/nettester.c index 566f553b..7f1a6b1f 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -38,6 +38,7 @@ #include <sys/socket.h> #include <sys/wait.h> #include <sys/stat.h> +#include <sys/time.h> #include <arpa/inet.h> #include <assert.h> #include <unistd.h> @@ -46,6 +47,7 @@ #include <signal.h> #include <netinet/in.h> #include <getopt.h> +#include <errno.h> #define EXIT_FAILURE 1 #define INVALID_SOCKET -1 @@ -60,7 +62,11 @@ static char *testSuite = NULL; /* name of current test suite */ static int iPort = 12514; /* port which shall be used for sending data */ static char* pszCustomConf = NULL; /* custom config file, use -c conf to specify */ static int verbose = 0; /* verbose output? -v option */ +static int IPv4Only = 0; /* use only IPv4 in rsyslogd call? */ +/* these two are quick hacks... */ +int iFailed = 0; +int iTests = 0; /* provide user-friednly name of input mode */ @@ -79,14 +85,33 @@ static char *inputMode2Str(inputMode_t mode) void readLine(int fd, char *ln) { + char *orig = ln; char c; int lenRead; + + if(verbose) + fprintf(stderr, "begin readLine\n"); lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + if(c == '\0') { + *ln = c; + fprintf(stderr, "Warning: there was a '\\0'-Byte in the read response " + "right after this string: '%s'\n", orig); + c = '?'; + } *ln++ = c; - lenRead = read(fd, &c, 1); + lenRead = read(fd, &c, 1); } *ln = '\0'; + + if(lenRead < 0) { + printf("read from rsyslogd returned with error '%s' - aborting test\n", strerror(errno)); + exit(1); + } + + if(verbose) + fprintf(stderr, "end readLine, val read '%s'\n", orig); } @@ -104,6 +129,7 @@ tcpSend(char *buf, int lenBuf) { static int sock = INVALID_SOCKET; struct sockaddr_in addr; + int retries; if(sock == INVALID_SOCKET) { /* first time, need to connect to target */ @@ -119,10 +145,20 @@ tcpSend(char *buf, int lenBuf) fprintf(stderr, "inet_aton() failed\n"); return(1); } - if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { - fprintf(stderr, "connect() failed\n"); - return(1); - } + retries = 0; + while(1) { /* loop broken inside */ + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == 0) { + break; + } else { + if(retries++ == 50) { + ++iFailed; + fprintf(stderr, "connect() failed\n"); + return(1); + } else { + usleep(100000); /* ms = 1000 us! */ + } + } + } } /* send test data */ @@ -186,19 +222,21 @@ int openPipe(char *configFile, pid_t *pid, int *pfd) int pipefd[2]; pid_t cpid; char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime/.libs:../.libs", NULL }; + "-M../runtime/.libs:../.libs", NULL, NULL}; char confFile[1024]; char *newenviron[] = { NULL }; /* debug aide... char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout", - "RSYSLOG_DEBUGLOG=tmp", NULL }; + "RSYSLOG_DEBUGLOG=log", NULL }; */ - sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, (pszCustomConf == NULL) ? configFile : pszCustomConf); newargv[1] = confFile; + if(IPv4Only) + newargv[(sizeof(newargv)/sizeof(char*)) - 2] = "-4"; + if (pipe(pipefd) == -1) { perror("pipe"); exit(EXIT_FAILURE); @@ -247,42 +285,58 @@ processTestFile(int fd, char *pszFileName) /* skip comments at start of file */ - getline(&testdata, &lenLn, fp); while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + /* this is not perfect, but works ;) */ + if(feof(fp)) + break; - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send (we could use function pointers here...) */ - if(inputMode == inputUDP) { - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - } else { - if(tcpSend(testdata, strlen(testdata)) != 0) - return(2); - } + ++iTests; /* increment test count, we now do one! */ - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send (we could use function pointers here...) */ + if(inputMode == inputUDP) { + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + } else { + if(tcpSend(testdata, strlen(testdata)) != 0) + return(2); + } - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strlen(buf) == 0) { + printf("something went wrong - read a zero-length string from rsyslogd\n"); + exit(1); + } + if(strcmp(expected, buf)) { + ++iFailed; + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + /* clean up after the try */ + free(testdata); + testdata = NULL; + free(expected); + expected = NULL; } - free(testdata); - free(expected); fclose(fp); return(ret); } @@ -297,8 +351,6 @@ processTestFile(int fd, char *pszFileName) int doTests(int fd, char *files) { - int iFailed = 0; - int iTests = 0; int ret; char *testFile; glob_t testFiles; @@ -313,7 +365,6 @@ doTests(int fd, char *files) if(stat((char*) testFile, &fileInfo) != 0) continue; /* continue with the next file if we can't stat() the file */ - ++iTests; /* all regular files are run through the test logic. Symlinks don't work. */ if(S_ISREG(fileInfo.st_mode)) { /* config file */ if(verbose) printf("processing test case '%s' ... ", testFile); @@ -321,8 +372,9 @@ doTests(int fd, char *files) if(ret == 0) { if(verbose) printf("successfully completed\n"); } else { - if(verbose) printf("failed!\n"); - ++iFailed; + if(!verbose) + printf("test '%s' ", testFile); + printf("failed!\n"); } } } @@ -338,11 +390,24 @@ doTests(int fd, char *files) return(iFailed); } + +/* indicate that our child has died (where it is not permitted to!). + */ +void childDied(__attribute__((unused)) int sig) +{ + printf("ERROR: child died unexpectedly (maybe a segfault?)!\n"); + exit(1); +} + + /* cleanup */ void doAtExit(void) { int status; + /* disarm died-child handler */ + signal(SIGCHLD, SIG_IGN); + if(rsyslogdPid != 0) { kill(rsyslogdPid, SIGTERM); waitpid(rsyslogdPid, &status, 0); /* wait until instance terminates */ @@ -365,8 +430,11 @@ int main(int argc, char *argv[]) char buf[4096]; char testcases[4096]; - while((opt = getopt(argc, argv, "c:i:p:t:v")) != EOF) { + while((opt = getopt(argc, argv, "4c:i:p:t:v")) != EOF) { switch((char)opt) { + case '4': + IPv4Only = 1; + break; case 'c': pszCustomConf = optarg; break; @@ -423,6 +491,9 @@ int main(int argc, char *argv[]) } fclose(fp); + /* arm died-child handler */ + signal(SIGCHLD, childDied); + /* start to be tested rsyslogd */ openPipe(testSuite, &rsyslogdPid, &fd); readLine(fd, buf); @@ -433,5 +504,6 @@ int main(int argc, char *argv[]) ret = 1; if(verbose) printf("End of nettester run (%d).\n", ret); + exit(ret); } diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index 2c2a8ef3..4e916f1e 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,4 +1,4 @@ -echo test omod-if-array via udp +echo \[omod-if-array.sh\]: test omod-if-array via udp $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tomod-if-array -iudp -p4711 diff --git a/tests/omruleset-queue.sh b/tests/omruleset-queue.sh new file mode 100755 index 00000000..cfb80c62 --- /dev/null +++ b/tests/omruleset-queue.sh @@ -0,0 +1,19 @@ +# test for omruleset. What we do is have the main queue forward +# all messages to a secondary ruleset via omruleset, which then does +# the actual file write. We check if all messages arrive at the file, +# what implies that omruleset works. No filters or special queue modes +# are used, but the ruleset uses its own queue. So we can also inject +# more messages without running into troubles. +# added 2009-11-02 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[omruleset-queue.sh\]: test for omruleset functionality with a ruleset queue +source $srcdir/diag.sh init +source $srcdir/diag.sh startup omruleset-queue.conf +source $srcdir/diag.sh injectmsg 0 20000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 19999 +source $srcdir/diag.sh exit diff --git a/tests/omruleset.sh b/tests/omruleset.sh new file mode 100755 index 00000000..dbc5cb31 --- /dev/null +++ b/tests/omruleset.sh @@ -0,0 +1,22 @@ +# Basic test for omruleset. What we do is have the main queue forward +# all messages to a secondary ruleset via omruleset, which then does +# the actual file write. We check if all messages arrive at the file, +# what implies that omruleset works. No filters or special queue modes +# are used, so the message is re-enqueued into the main message queue. +# We inject just 5,000 message because we may otherwise run into +# queue full conditions (as we use the same queue) and that +# would result in longer execution time. In any case, 5000 messages +# are well enough to test what we want to test. +# added 2009-11-02 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[omruleset.sh\]: basic test for omruleset functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup omruleset.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/ourtail.c b/tests/ourtail.c index f2751c72..4e8a6412 100644 --- a/tests/ourtail.c +++ b/tests/ourtail.c @@ -28,7 +28,7 @@ */ #include <stdio.h> -int main(int argc, char *argv[]) +int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) { int c; @@ -40,4 +40,6 @@ int main(int argc, char *argv[]) for( ; c != EOF ; c = getchar()) putchar(c); + + return 0; } diff --git a/tests/parsertest.sh b/tests/parsertest.sh index afdb9469..06fcc8d6 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,13 +1,25 @@ -echo test parsertest via udp -$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason +echo TEST: \[parsertest.sh\]: various parser tests +source $srcdir/diag.sh init +source $srcdir/diag.sh nettester parse1 udp +source $srcdir/diag.sh nettester parse1 tcp +source $srcdir/diag.sh nettester parse2 udp +source $srcdir/diag.sh nettester parse2 tcp +source $srcdir/diag.sh nettester parse_8bit_escape udp +source $srcdir/diag.sh nettester parse_8bit_escape tcp +source $srcdir/diag.sh nettester parse3 udp +source $srcdir/diag.sh nettester parse3 tcp +source $srcdir/diag.sh nettester parse_invld_regex udp +source $srcdir/diag.sh nettester parse_invld_regex tcp -./nettester -tparse1 -iudp -if [ "$?" -ne "0" ]; then - exit 1 -fi - -echo test parsertest via tcp -./nettester -tparse1 -itcp -if [ "$?" -ne "0" ]; then - exit 1 -fi +echo \[parsertest.sh]: redoing tests in IPv4-only mode +source $srcdir/diag.sh nettester parse1 udp -4 +source $srcdir/diag.sh nettester parse1 tcp -4 +source $srcdir/diag.sh nettester parse2 udp -4 +source $srcdir/diag.sh nettester parse2 tcp -4 +source $srcdir/diag.sh nettester parse_8bit_escape udp -4 +source $srcdir/diag.sh nettester parse_8bit_escape tcp -4 +source $srcdir/diag.sh nettester parse3 udp -4 +source $srcdir/diag.sh nettester parse3 tcp -4 +source $srcdir/diag.sh nettester parse_invld_regex udp -4 +source $srcdir/diag.sh nettester parse_invld_regex tcp -4 +source $srcdir/diag.sh exit diff --git a/tests/pipeaction.sh b/tests/pipeaction.sh new file mode 100755 index 00000000..26a4c22a --- /dev/null +++ b/tests/pipeaction.sh @@ -0,0 +1,33 @@ +# Test for the pipe output action. +# will create a fifo in the current directory, write to it and +# then do the usual sequence checks. +# added 2009-11-05 by RGerhards +echo =============================================================================== +echo \[pipeaction.sh\]: testing pipe output action + +# create the pipe and start a background process that copies data from +# it to the "regular" work file +source $srcdir/diag.sh init +rm -f rsyslog-testbench-fifo +mkfifo rsyslog-testbench-fifo +cp rsyslog-testbench-fifo rsyslog.out.log & +CPPROCESS=$! +echo background cp process id is $CPPROCESS + +# now do the usual run +source $srcdir/diag.sh startup pipeaction.conf +# 20000 messages should be enough +#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000 +source $srcdir/diag.sh injectmsg 0 20000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown + +# wait for the cp process to finish, do pipe-specific cleanup +echo waiting for background cp to terminate... +wait $CPPROCESS +rm -f rsyslog-testbench-fifo +echo background cp has terminated, continue test... + +# and continue the usual checks +source $srcdir/diag.sh seq-check 0 19999 +source $srcdir/diag.sh exit diff --git a/tests/proprepltest.sh b/tests/proprepltest.sh new file mode 100755 index 00000000..2e59a31d --- /dev/null +++ b/tests/proprepltest.sh @@ -0,0 +1,7 @@ +echo \[proprepltest.sh\]: various tests for the property replacer +source $srcdir/diag.sh init +source $srcdir/diag.sh nettester rfctag udp +source $srcdir/diag.sh nettester rfctag tcp +source $srcdir/diag.sh nettester nolimittag udp +source $srcdir/diag.sh nettester nolimittag tcp +source $srcdir/diag.sh init diff --git a/tests/queue-persist-drvr.sh b/tests/queue-persist-drvr.sh new file mode 100755 index 00000000..53fbcb8b --- /dev/null +++ b/tests/queue-persist-drvr.sh @@ -0,0 +1,29 @@ +# Test for queue data persisting at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo testing memory queue persisting to disk, mode $1 +source $srcdir/diag.sh init + +# prepare config +echo \$MainMsgQueueType $1 > work-queuemode.conf +echo "*.* :omtesting:sleep 0 1000" > work-delay.conf + +# inject 5000 msgs, so that we do not hit the high watermark +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh injectmsg 0 5000 +$srcdir/diag.sh shutdown-immediate +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh check-mainq-spool + +# restart engine and have rest processed +#remove delay +echo "#" > work-delay.conf +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/queue-persist.sh b/tests/queue-persist.sh new file mode 100755 index 00000000..ff1842bb --- /dev/null +++ b/tests/queue-persist.sh @@ -0,0 +1,12 @@ +# Test for queue data persisting at shutdown. We use the actual driver +# to carry out multiple tests with different queue modes +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo \[queue-persist.sh\]: +source $srcdir/queue-persist-drvr.sh LinkedList +source $srcdir/queue-persist-drvr.sh FixedArray +# the disk test should not fail, however, the config is extreme and using +# it more or less is a config error +source $srcdir/queue-persist-drvr.sh Disk +# we do not test Direct mode because this absolute can not work in direct mode +# (maybe we should do a fail-type of test?) diff --git a/tests/rscript.c b/tests/rscript.c index ce81491c..6361aec4 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -104,8 +104,8 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { printf("error: compiled program different from expected result!\n"); - printf("generated vmprg (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); - printf("expected (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } diff --git a/tests/rsf_getenv.sh b/tests/rsf_getenv.sh new file mode 100755 index 00000000..42de20fe --- /dev/null +++ b/tests/rsf_getenv.sh @@ -0,0 +1,17 @@ +# Test for the getenv() rainerscript function +# this is a quick test, but it gurantees that the code path is +# at least progressed (but we do not check for unset envvars!) +# added 2009-11-03 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =============================================================================== +echo \[rsf_getenv.sh\]: testing RainerScript getenv\(\) function +export MSGNUM="msgnum:" +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rsf_getenv.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 9999 +unset MSGNUM +source $srcdir/diag.sh exit diff --git a/tests/rt-init.c b/tests/rt-init.c index aaac7ed1..b9c4ce2e 100644 --- a/tests/rt-init.c +++ b/tests/rt-init.c @@ -21,10 +21,9 @@ * * A copy of the GPL can be found in the file "COPYING" in this distribution. */ -#include <stdio.h> - #include "rsyslog.h" #include "testbench.h" +#include <stdio.h> /* must be last, else we get a zlib compile error on some platforms */ MODULE_TYPE_TESTBENCH diff --git a/tests/rulesetmultiqueue.sh b/tests/rulesetmultiqueue.sh new file mode 100755 index 00000000..aff55148 --- /dev/null +++ b/tests/rulesetmultiqueue.sh @@ -0,0 +1,33 @@ +# Test for disk-only queue mode +# This tests defines three rulesets, each one with its own queue. Then, it +# sends data to them and checks the outcome. Note that we do need to +# use some custom code as the test driver framework does not (yet?) +# support multi-output-file operations. +# added 2009-10-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[rulesetmultiqueu.sh\]: testing multiple queues via rulesets +source $srcdir/diag.sh init +rm -f rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log +source $srcdir/diag.sh startup rulesetmultiqueue.conf +source $srcdir/diag.sh wait-startup +# now fill the three files (a bit sequentially, but they should +# still get their share of concurrency - to increase the chance +# we use three connections per set). +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 3 20000 0 +source $srcdir/diag.sh tcpflood 127.0.0.1 13515 3 20000 20000 +source $srcdir/diag.sh tcpflood 127.0.0.1 13516 3 20000 40000 + +# in this version of the imdiag, we do not have the capability to poll +# all queues for emptyness. So we do a sleep in the hopes that this will +# sufficiently drain the queues. This is race, but the best we currently +# can do... - rgerhards, 2009-11-05 +sleep 2 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +# now consolidate all logs into a single one so that we can use the +# regular check logic +cat rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 59999 +rm -f rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log +source $srcdir/diag.sh exit diff --git a/tests/runtime-dummy.c b/tests/runtime-dummy.c index 9cddd913..5a9039bf 100644 --- a/tests/runtime-dummy.c +++ b/tests/runtime-dummy.c @@ -25,17 +25,22 @@ * * A copy of the GPL can be found in the file "COPYING" in this distribution. */ +#include "config.h" #include <stdlib.h> +#include "rsyslog.h" int bReduceRepeatMsgs = 0; int repeatinterval = 30; int bActExecWhenPrevSusp = 0; int iActExecOnceInterval = 1; int MarkInterval = 30; +void *pMsgQueue = NULL; void cflineClassic(void) {}; void selectorAddList(void) {}; void selectorConstruct(void) {}; void selectorDestruct(void) {}; +rsRetVal createMainQueue(void) { return RS_RET_ERR; } +ruleset_t *pCurrRuleset; /* these are required by some dynamically loaded modules */ diff --git a/tests/sndrcv.sh b/tests/sndrcv.sh new file mode 100755 index 00000000..2fc3bd82 --- /dev/null +++ b/tests/sndrcv.sh @@ -0,0 +1,9 @@ +# This tests two rsyslog instances. Instance +# TWO sends data to instance ONE. A number of messages is injected into +# the instance 2 and we finally check if all those messages +# arrived at instance 1. +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv.sh\]: testing sending and receiving via tcp +source $srcdir/sndrcv_drvr.sh sndrcv 50000 diff --git a/tests/sndrcv_drvr.sh b/tests/sndrcv_drvr.sh new file mode 100755 index 00000000..63afd6b5 --- /dev/null +++ b/tests/sndrcv_drvr.sh @@ -0,0 +1,50 @@ +# This is test driver for testing two rsyslog instances. It can be +# utilized by any test that just needs two instances with different +# config files, where messages are injected in instance TWO and +# (with whatever rsyslog mechanism) being relayed over to instance ONE, +# where they are written to the log file. After the run, the completeness +# of that log file is checked. +# The code is almost the same, but the config files differ (probably greatly) +# for different test cases. As such, this driver needs to be called with the +# config file name ($2). From that name, the sender and receiver config file +# names are automatically generated. +# So: $1 config file name, $2 number of messages +# +# A note on TLS testing: the current testsuite (in git!) already contains +# TLS test cases. However, getting these test cases correct is not simple. +# That's not a problem with the code itself, but rater a problem with +# synchronization in the test environment. So I have deciced to keep the +# TLS tests in, but not yet actually utilize them. This is most probably +# left as an excercise for future (devel) releases. -- rgerhards, 2009-11-11 +# +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +source $srcdir/diag.sh init +# start up the instances +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="log" +source $srcdir/diag.sh startup $1_rcvr.conf +source $srcdir/diag.sh wait-startup +#export RSYSLOG_DEBUGLOG="log2" +#valgrind="valgrind" +source $srcdir/diag.sh startup $1_sender.conf 2 +source $srcdir/diag.sh wait-startup 2 +# may be needed by TLS (once we do it): sleep 30 + +# now inject the messages into instance 2. It will connect to instance 1, +# and that instance will record the data. +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 $2 1 +sleep 2 # make sure all data is received in input buffers +# shut down sender when everything is sent, receiver continues to run concurrently +# may be needed by TLS (once we do it): sleep 60 +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 +# now it is time to stop the receiver as well +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown + +# may be needed by TLS (once we do it): sleep 60 +# do the final check +source $srcdir/diag.sh seq-check 1 $2 +source $srcdir/diag.sh exit diff --git a/tests/sndrcv_gzip.sh b/tests/sndrcv_gzip.sh new file mode 100755 index 00000000..4931f3d0 --- /dev/null +++ b/tests/sndrcv_gzip.sh @@ -0,0 +1,7 @@ +# This test is similar to tcpsndrcv, but it forwards messages in +# zlib-compressed format (our own syslog extension). +# rgerhards, 2009-11-11 +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_gzip.sh\]: testing sending and receiving via tcp in zlib mode +source $srcdir/sndrcv_drvr.sh sndrcv_gzip 50000 diff --git a/tests/sndrcv_omudpspoof.sh b/tests/sndrcv_omudpspoof.sh new file mode 100755 index 00000000..bb804d93 --- /dev/null +++ b/tests/sndrcv_omudpspoof.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the standard +# ports. Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_omudpspoof.sh\]: testing sending and receiving via omudp +source $srcdir/sndrcv_drvr.sh sndrcv_omudpspoof 50 diff --git a/tests/sndrcv_omudpspoof_nonstdpt.sh b/tests/sndrcv_omudpspoof_nonstdpt.sh new file mode 100755 index 00000000..6aeb1a5f --- /dev/null +++ b/tests/sndrcv_omudpspoof_nonstdpt.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the standard +# ports. Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_omudpspoof_nonstdpt.sh\]: testing sending and receiving via omudp +source $srcdir/sndrcv_drvr.sh sndrcv_omudpspoof_nonstdpt 50 diff --git a/tests/sndrcv_udp.sh b/tests/sndrcv_udp.sh new file mode 100755 index 00000000..274a414a --- /dev/null +++ b/tests/sndrcv_udp.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the standard +# ports. Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_udp.sh\]: testing sending and receiving via udp +source $srcdir/sndrcv_drvr.sh sndrcv_udp 50 diff --git a/tests/sndrcv_udp_nonstdpt.sh b/tests/sndrcv_udp_nonstdpt.sh new file mode 100755 index 00000000..2ad2906f --- /dev/null +++ b/tests/sndrcv_udp_nonstdpt.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the non-standard port 2514 +# Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_udp_nonstdpt.sh\]: testing sending and receiving via udp +source $srcdir/sndrcv_drvr.sh sndrcv_udp_nonstdpt 50 diff --git a/tests/tabescape_dflt.sh b/tests/tabescape_dflt.sh new file mode 100755 index 00000000..d0e13ec9 --- /dev/null +++ b/tests/tabescape_dflt.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_dflt.sh\]: test for default tab escaping +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_dflt -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_dflt -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/tabescape_off.sh b/tests/tabescape_off.sh new file mode 100755 index 00000000..71ede7c0 --- /dev/null +++ b/tests/tabescape_off.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_off.sh\]: test for tab escaping off +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_off -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_off -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/tcpflood.c b/tests/tcpflood.c index c3c9c871..0439e33e 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -6,6 +6,7 @@ * argv[2] target port * argv[3] number of connections * argv[4] number of messages to send (connection is random) + * argv[5] initial message number (optional) * * Part of the testbench for rsyslog. * @@ -51,6 +52,7 @@ static int targetPort; static int numMsgsToSend; /* number of messages to send */ static int numConnections; /* number of connections to create */ static int *sockArray; /* array of sockets to use */ +static int msgNum = 0; /* initial message number to start with */ /* open a single tcp connection @@ -59,6 +61,7 @@ int openConn(int *fd) { int sock; struct sockaddr_in addr; + int retries = 0; if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { perror("socket()"); @@ -72,11 +75,19 @@ int openConn(int *fd) fprintf(stderr, "inet_aton() failed\n"); return(1); } - if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { - perror("connect()"); - fprintf(stderr, "connect() failed\n"); - return(1); - } + while(1) { /* loop broken inside */ + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == 0) { + break; + } else { + if(retries++ == 50) { + perror("connect()"); + fprintf(stderr, "connect() failed\n"); + return(1); + } else { + usleep(100000); /* ms = 1000 us! */ + } + } + } *fd = sock; return 0; @@ -166,19 +177,20 @@ int sendMessages(void) socknum = i - (numMsgsToSend - numConnections); else socknum = rand() % numConnections; - lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", msgNum); lenSend = send(sockArray[socknum], buf, lenBuf, 0); if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); perror("send test data"); - printf("send() failed at socket %d, index %d\n", socknum, i); + printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum); fflush(stderr); return(1); } if(i % 100 == 0) { printf("\r%5.5d", i); } + ++msgNum; } printf("\r%5.5d messages sent\n", i); @@ -260,9 +272,9 @@ int main(int argc, char *argv[]) setvbuf(stdout, buf, _IONBF, 48); - if(argc != 5) { + if(argc != 5 && argc != 6) { printf("Invalid call of tcpflood\n"); - printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages [initial msgnum]\n"); exit(1); } @@ -270,6 +282,8 @@ int main(int argc, char *argv[]) targetPort = atoi(argv[2]); numConnections = atoi(argv[3]); numMsgsToSend = atoi(argv[4]); + if(argc == 6) + msgNum = atoi(argv[5]); if(openConnections() != 0) { printf("error opening connections\n"); diff --git a/tests/testsuites/1.retry.conf b/tests/testsuites/1.retry.conf new file mode 100644 index 00000000..c464b19c --- /dev/null +++ b/tests/testsuites/1.retry.conf @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, diff --git a/tests/testsuites/1.tabescape_dflt b/tests/testsuites/1.tabescape_dflt new file mode 100644 index 00000000..91444bd3 --- /dev/null +++ b/tests/testsuites/1.tabescape_dflt @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT#011after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.tabescape_off b/tests/testsuites/1.tabescape_off new file mode 100644 index 00000000..6a331c35 --- /dev/null +++ b/tests/testsuites/1.tabescape_off @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/4.parse1 b/tests/testsuites/4.parse1 new file mode 100644 index 00000000..07e2445a --- /dev/null +++ b/tests/testsuites/4.parse1 @@ -0,0 +1,4 @@ +<29>Jul 31 21:39:21 example-b example-gw[10538]: disconnect host=/192.0.2.1 destination=192.0.2.2/11282 in=3274 out=1448 duration=0 +29,daemon,notice,Jul 31 21:39:21,example-b,example-gw,example-gw[10538]:, disconnect host=/192.0.2.1 destination=192.0.2.2/11282 in=3274 out=1448 duration=0 +# yet another real-life sample where we had some issues with - the important +# part is the dash inside the hostname! diff --git a/tests/testsuites/8bit.parse1 b/tests/testsuites/8bit.parse1 new file mode 100644 index 00000000..90db6352 --- /dev/null +++ b/tests/testsuites/8bit.parse1 @@ -0,0 +1,2 @@ +<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü +6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: äöü diff --git a/tests/testsuites/8bit.parse_8bit_escape b/tests/testsuites/8bit.parse_8bit_escape new file mode 100644 index 00000000..b2f6335c --- /dev/null +++ b/tests/testsuites/8bit.parse_8bit_escape @@ -0,0 +1,2 @@ +<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü +6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: #303#244#303#266#303#274 diff --git a/tests/testsuites/Apr.ts3164 b/tests/testsuites/Apr.ts3164 new file mode 100644 index 00000000..3134f224 --- /dev/null +++ b/tests/testsuites/Apr.ts3164 @@ -0,0 +1,3 @@ +<167>Apr 6 16:57:54 172.20.245.8 TAG: MSG +Apr 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Aug.ts3164 b/tests/testsuites/Aug.ts3164 new file mode 100644 index 00000000..d9a721eb --- /dev/null +++ b/tests/testsuites/Aug.ts3164 @@ -0,0 +1,3 @@ +<167>Aug 6 16:57:54 172.20.245.8 TAG: MSG +Aug 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Dec.ts3164 b/tests/testsuites/Dec.ts3164 new file mode 100644 index 00000000..080ba401 --- /dev/null +++ b/tests/testsuites/Dec.ts3164 @@ -0,0 +1,3 @@ +<167>Dec 6 16:57:54 172.20.245.8 TAG: MSG +Dec 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Feb.ts3164 b/tests/testsuites/Feb.ts3164 new file mode 100644 index 00000000..d1eaaa33 --- /dev/null +++ b/tests/testsuites/Feb.ts3164 @@ -0,0 +1,3 @@ +<167>Feb 6 16:57:54 172.20.245.8 TAG: MSG +Feb 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jan.ts3164 b/tests/testsuites/Jan.ts3164 new file mode 100644 index 00000000..0cb1c8e2 --- /dev/null +++ b/tests/testsuites/Jan.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 6 16:57:54 172.20.245.8 TAG: MSG +Jan 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jul.ts3164 b/tests/testsuites/Jul.ts3164 new file mode 100644 index 00000000..562e1ec4 --- /dev/null +++ b/tests/testsuites/Jul.ts3164 @@ -0,0 +1,3 @@ +<167>Jul 6 16:57:54 172.20.245.8 TAG: MSG +Jul 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jun.ts3164 b/tests/testsuites/Jun.ts3164 new file mode 100644 index 00000000..ede27e0e --- /dev/null +++ b/tests/testsuites/Jun.ts3164 @@ -0,0 +1,3 @@ +<167>Jun 6 16:57:54 172.20.245.8 TAG: MSG +Jun 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Mar.ts3164 b/tests/testsuites/Mar.ts3164 new file mode 100644 index 00000000..55dd5bc2 --- /dev/null +++ b/tests/testsuites/Mar.ts3164 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 TAG: MSG +Mar 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/May.ts3164 b/tests/testsuites/May.ts3164 new file mode 100644 index 00000000..72a5a301 --- /dev/null +++ b/tests/testsuites/May.ts3164 @@ -0,0 +1,3 @@ +<167>May 6 16:57:54 172.20.245.8 TAG: MSG +May 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Nov.ts3164 b/tests/testsuites/Nov.ts3164 new file mode 100644 index 00000000..e8f00e01 --- /dev/null +++ b/tests/testsuites/Nov.ts3164 @@ -0,0 +1,3 @@ +<167>Nov 6 16:57:54 172.20.245.8 TAG: MSG +Nov 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Oct.ts3164 b/tests/testsuites/Oct.ts3164 new file mode 100644 index 00000000..01423fef --- /dev/null +++ b/tests/testsuites/Oct.ts3164 @@ -0,0 +1,3 @@ +<167>Oct 6 16:57:54 172.20.245.8 TAG: MSG +Oct 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Sep.ts3164 b/tests/testsuites/Sep.ts3164 new file mode 100644 index 00000000..6c9e48e0 --- /dev/null +++ b/tests/testsuites/Sep.ts3164 @@ -0,0 +1,3 @@ +<167>Sep 6 16:57:54 172.20.245.8 TAG: MSG +Sep 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/arrayqueue.conf b/tests/testsuites/arrayqueue.conf new file mode 100644 index 00000000..c5874a83 --- /dev/null +++ b/tests/testsuites/arrayqueue.conf @@ -0,0 +1,14 @@ +# Test for queue fixedArray mode (see .sh file for details) +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +# set spool locations and switch queue to disk-only mode +$MainMsgQueueType FixedArray + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/badqi.conf b/tests/testsuites/badqi.conf new file mode 100644 index 00000000..0ab059df --- /dev/null +++ b/tests/testsuites/badqi.conf @@ -0,0 +1,15 @@ +# Test for bad .qi file (see .sh file for details) +# rgerhards, 2009-10-21 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +# instruct to use bad .qi file +$WorkDirectory bad_qi +$ActionQueueType LinkedList +$ActionQueueFileName dbq +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf new file mode 100644 index 00000000..843a3e4f --- /dev/null +++ b/tests/testsuites/da-mainmsg-q.conf @@ -0,0 +1,21 @@ +# Test for DA mode in main message queue (see .sh file for details) +# rgerhards, 2009-04-22 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk assisted mode +$WorkDirectory test-spool +$MainMsgQueueSize 200 # this *should* trigger moving on to DA mode... +# note: we must set QueueSize sufficiently high, so that 70% (light delay mark) +# is high enough above HighWatermark! +$MainMsgQueueHighWatermark 80 +$MainMsgQueueLowWatermark 40 +$MainMsgQueueFilename mainq +$MainMsgQueueType linkedlist + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf new file mode 100644 index 00000000..9e9e28fe --- /dev/null +++ b/tests/testsuites/diag-common.conf @@ -0,0 +1,16 @@ +# This is a config include file. It sets up rsyslog so that the +# diag system can successfully be used. Also, it generates a file +# "rsyslogd.started" after rsyslogd is initialized. This config file +# should be included in all tests that intend to use common code for +# controlling the daemon. +# NOTE: we assume that rsyslogd's current working directory is +# ./tests (or the distcheck equivalent), in particlular that this +# config file resides in the testsuites subdirectory. +# rgerhards, 2009-05-27 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + +$template startupfile,"rsyslogd.started" # trick to use relative path names! +:syslogtag, contains, "rsyslogd" ?startupfile + +$ErrorMessagesToStderr off diff --git a/tests/testsuites/dircreate_dflt.conf b/tests/testsuites/dircreate_dflt.conf new file mode 100644 index 00000000..9b9aadb8 --- /dev/null +++ b/tests/testsuites/dircreate_dflt.conf @@ -0,0 +1,11 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/dircreate_off.conf b/tests/testsuites/dircreate_off.conf new file mode 100644 index 00000000..28ccbd8c --- /dev/null +++ b/tests/testsuites/dircreate_off.conf @@ -0,0 +1,12 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$CreateDirs off +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/discard.conf b/tests/testsuites/discard.conf new file mode 100644 index 00000000..bbe2fe77 --- /dev/null +++ b/tests/testsuites/discard.conf @@ -0,0 +1,13 @@ +# Test for discard functionality +# rgerhards, 2009-07-30 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +:msg, contains, "00000001" ~ + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/diskqueue-fsync.conf b/tests/testsuites/diskqueue-fsync.conf new file mode 100644 index 00000000..0a02c6ce --- /dev/null +++ b/tests/testsuites/diskqueue-fsync.conf @@ -0,0 +1,17 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueSyncQueueFiles on +$MainMsgQueueTimeoutShutdown 10000 +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf index 017ee96d..a992c5a5 100644 --- a/tests/testsuites/diskqueue.conf +++ b/tests/testsuites/diskqueue.conf @@ -1,14 +1,11 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - # set spool locations and switch queue to disk-only mode $WorkDirectory test-spool $MainMsgQueueFilename mainq diff --git a/tests/testsuites/empty.parse1 b/tests/testsuites/empty.parse1 new file mode 100644 index 00000000..86a86986 --- /dev/null +++ b/tests/testsuites/empty.parse1 @@ -0,0 +1,3 @@ +<14>Jan 6 2009 15:22:26 localhost +14,user,info,Jan 6 15:22:26,localhost,,, +#Note: there is one space after localhost, but then \n! diff --git a/tests/testsuites/execonlyonce.conf b/tests/testsuites/execonlyonce.conf new file mode 100644 index 00000000..085b970e --- /dev/null +++ b/tests/testsuites/execonlyonce.conf @@ -0,0 +1,12 @@ +# see the equally-named .sh file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +$ActionExecOnlyOnceEveryInterval 3 +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/execonlyonce.data b/tests/testsuites/execonlyonce.data new file mode 100644 index 00000000..3c54f3d4 --- /dev/null +++ b/tests/testsuites/execonlyonce.data @@ -0,0 +1,2 @@ +00000001 +00000100 diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf index ec059fe4..ccdc15fb 100644 --- a/tests/testsuites/imtcp-multiport.conf +++ b/tests/testsuites/imtcp-multiport.conf @@ -1,16 +1,13 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-05-22 +$IncludeConfig diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 $InputTCPServerRun 13515 $InputTCPServerRun 13516 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! :msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/invalid.conf b/tests/testsuites/invalid.conf new file mode 100644 index 00000000..8a865ba5 --- /dev/null +++ b/tests/testsuites/invalid.conf @@ -0,0 +1,3 @@ +# This is an invalid config file that shall trigger an exit code +# with the config verification run +$invalid diff --git a/tests/testsuites/linkedlistqueue.conf b/tests/testsuites/linkedlistqueue.conf new file mode 100644 index 00000000..92a9649c --- /dev/null +++ b/tests/testsuites/linkedlistqueue.conf @@ -0,0 +1,16 @@ +# Test for queue LinkedList mode (see .sh file for details) +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$MainMsgQueueType LinkedList + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/malformed1.parse1 b/tests/testsuites/malformed1.parse1 new file mode 100644 index 00000000..a8825fe8 --- /dev/null +++ b/tests/testsuites/malformed1.parse1 @@ -0,0 +1,6 @@ +<131>Oct 8 23:05:06 10.321.1.123 05",result_code=200,b +131,local0,err,Oct 8 23:05:06,10.321.1.123,05",result_code=200,b,05",result_code=200,b, +# a somewhat mangeld-with real-life sample of a malformed message +# the key here is not what is being parsed, but that we do not abort! +# NOTE: if a parser enhancement breaks the format, this is probably OK +# also note that the above message does NOT contain a MSG part diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index 3867da46..eb9db257 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,16 +1,13 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 +$IncludeConfig diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $MaxOpenFiles 2000 $InputTCPMaxSessions 1100 $InputTCPServerRun 13514 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! :msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/mark.parse1 b/tests/testsuites/mark.parse1 new file mode 100644 index 00000000..fff9ae6d --- /dev/null +++ b/tests/testsuites/mark.parse1 @@ -0,0 +1,7 @@ +#This is a malformed message, but one from real life. At least, +#it should be parsed as can be seen here. +<6>Feb 18 16:01:59 serverX -- MARK -- +6,kern,info,Feb 18 16:01:59,serverX,--,--, MARK -- +# and the next one as an extreme case (note the absence of PRI) +Feb 18 16:01:59 serverX -- MARK -- +13,user,notice,Feb 18 16:01:59,serverX,--,--, MARK -- diff --git a/tests/testsuites/master.nolimittag b/tests/testsuites/master.nolimittag new file mode 100644 index 00000000..502d9d5d --- /dev/null +++ b/tests/testsuites/master.nolimittag @@ -0,0 +1,11 @@ +<167>Mar 6 16:57:54 172.20.245.8 TAG: Rest of message... ++TAG:+ +# now one char, no colon +<167>Mar 6 16:57:54 172.20.245.8 0 Rest of message... ++0+ +# Now exactly with 32 characters +<167>Mar 6 16:57:54 172.20.245.8 01234567890123456789012345678901 Rest of message... ++01234567890123456789012345678901+ +# Now oversize, should be completely output with this config +<167>Mar 6 16:57:54 172.20.245.8 01234567890123456789012345678901-toolong Rest of message... ++01234567890123456789012345678901-toolong+ diff --git a/tests/testsuites/master.rfctag b/tests/testsuites/master.rfctag new file mode 100644 index 00000000..3f1e0c66 --- /dev/null +++ b/tests/testsuites/master.rfctag @@ -0,0 +1,11 @@ +<167>Mar 6 16:57:54 172.20.245.8 TAG: Rest of message... ++TAG:+ +# now one char, no colon +<167>Mar 6 16:57:54 172.20.245.8 0 Rest of message... ++0+ +# Now exactly with 32 characters +<167>Mar 6 16:57:54 172.20.245.8 01234567890123456789012345678901 Rest of message... ++01234567890123456789012345678901+ +# Now oversize, should be truncated with this config +<167>Mar 6 16:57:54 172.20.245.8 01234567890123456789012345678901-toolong Rest of message... ++01234567890123456789012345678901+ diff --git a/tests/testsuites/master.subsecond b/tests/testsuites/master.subsecond new file mode 100644 index 00000000..ee924877 --- /dev/null +++ b/tests/testsuites/master.subsecond @@ -0,0 +1,8 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +003 +# full precision +<34>1 2003-01-23T12:34:56.123456Z mymachine.example.com su - ID47 - MSG +123456 +# without +<34>1 2003-01-23T12:34:56Z mymachine.example.com su - ID47 - MSG +0 diff --git a/tests/testsuites/master.ts3339 b/tests/testsuites/master.ts3339 new file mode 100644 index 00000000..b4dd5f39 --- /dev/null +++ b/tests/testsuites/master.ts3339 @@ -0,0 +1,22 @@ +<34>1 2003-11-11T22:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T22:14:15.003Z +# next test +<34>1 2003-01-11T22:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-01-11T22:14:15.003Z +# next test +<34>1 2003-11-01T22:04:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-01T22:04:15.003Z +# next test +<34>1 2003-11-11T02:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T02:14:15.003Z +# next test +<34>1 2003-11-11T22:04:05.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003Z +# next test +<34>1 2003-11-11T22:04:05.003+02:00 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003+02:00 +# next test +<34>1 2003-11-11T22:04:05.003+01:30 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003+01:30 +<34>1 2003-11-11T22:04:05.123456+01:30 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.123456+01:30 diff --git a/tests/testsuites/master.tsmysql b/tests/testsuites/master.tsmysql new file mode 100644 index 00000000..dc6d85be --- /dev/null +++ b/tests/testsuites/master.tsmysql @@ -0,0 +1,2 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +20030123123456 diff --git a/tests/testsuites/master.tspgsql b/tests/testsuites/master.tspgsql new file mode 100644 index 00000000..d7ac19ff --- /dev/null +++ b/tests/testsuites/master.tspgsql @@ -0,0 +1,2 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +2003-01-23 12:34:56 diff --git a/tests/testsuites/mon1digit.ts3164 b/tests/testsuites/mon1digit.ts3164 new file mode 100644 index 00000000..0cb1c8e2 --- /dev/null +++ b/tests/testsuites/mon1digit.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 6 16:57:54 172.20.245.8 TAG: MSG +Jan 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/mon2digit.ts3164 b/tests/testsuites/mon2digit.ts3164 new file mode 100644 index 00000000..9606961c --- /dev/null +++ b/tests/testsuites/mon2digit.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 16 16:57:54 172.20.245.8 TAG: MSG +Jan 16 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/nolimittag.conf b/tests/testsuites/nolimittag.conf new file mode 100644 index 00000000..0b6ec387 --- /dev/null +++ b/tests/testsuites/nolimittag.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"+%syslogtag%+\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/omruleset-queue.conf b/tests/testsuites/omruleset-queue.conf new file mode 100644 index 00000000..edc33c40 --- /dev/null +++ b/tests/testsuites/omruleset-queue.conf @@ -0,0 +1,20 @@ +# test for omruleset (see .sh file for details) +# rgerhards, 2009-11-02 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/omruleset/.libs/omruleset +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +$ruleset rsinclude +# create ruleset main queue with default parameters +$RulesetCreateMainQueue on +# make sure we do not terminate too early! +$MainMsgQueueTimeoutShutdown 10000 +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +$ruleset RSYSLOG_DefaultRuleset +$ActionOmrulesetRulesetName rsinclude +*.* :omruleset: diff --git a/tests/testsuites/omruleset.conf b/tests/testsuites/omruleset.conf new file mode 100644 index 00000000..fa6b906e --- /dev/null +++ b/tests/testsuites/omruleset.conf @@ -0,0 +1,16 @@ +# Basic test for omruleset (see .sh file for details) +# rgerhards, 2009-11-02 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/omruleset/.libs/omruleset +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +$ruleset rsinclude +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +$ruleset RSYSLOG_DefaultRuleset +$ActionOmrulesetRulesetName rsinclude +*.* :omruleset: diff --git a/tests/testsuites/oversizeTag-1.parse1 b/tests/testsuites/oversizeTag-1.parse1 new file mode 100644 index 00000000..d45ba1f2 --- /dev/null +++ b/tests/testsuites/oversizeTag-1.parse1 @@ -0,0 +1,2 @@ +<38>Mar 27 19:06:53 source_server 0123456789012345678901234567890123456789: MSG part +38,auth,info,Mar 27 19:06:53,source_server,0123456789012345678901234567890123456789,0123456789012345678901234567890123456789:, MSG part diff --git a/tests/testsuites/parse1udp.conf b/tests/testsuites/parse1udp.conf new file mode 100644 index 00000000..0fb7d16d --- /dev/null +++ b/tests/testsuites/parse1udp.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 12514 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/parse2.conf b/tests/testsuites/parse2.conf new file mode 100644 index 00000000..04d910bc --- /dev/null +++ b/tests/testsuites/parse2.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template output,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;output diff --git a/tests/testsuites/parse3.conf b/tests/testsuites/parse3.conf new file mode 100644 index 00000000..d5cf77d9 --- /dev/null +++ b/tests/testsuites/parse3.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9]+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" +*.* :omstdout:;output diff --git a/tests/testsuites/parse_8bit_escape.conf b/tests/testsuites/parse_8bit_escape.conf new file mode 100644 index 00000000..0598f33f --- /dev/null +++ b/tests/testsuites/parse_8bit_escape.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off +$Escape8BitCharactersOnReceive on + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/parse_invld_regex.conf b/tests/testsuites/parse_invld_regex.conf new file mode 100644 index 00000000..736aae59 --- /dev/null +++ b/tests/testsuites/parse_invld_regex.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" +*.* :omstdout:;output diff --git a/tests/testsuites/pipeaction.conf b/tests/testsuites/pipeaction.conf new file mode 100644 index 00000000..f58b6d65 --- /dev/null +++ b/tests/testsuites/pipeaction.conf @@ -0,0 +1,16 @@ +# Test for pipe output action (see .sh file for details) +# rgerhards, 2009-11-05 +$IncludeConfig diag-common.conf + +$MainMsgQueueTimeoutShutdown 10000 + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +# with pipes, we do not need to use absolute path names, so +# we can simply refer to our working pipe via the usual relative +# path name +:msg, contains, "msgnum:" |rsyslog-testbench-fifo;outfmt diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf new file mode 100644 index 00000000..8903042d --- /dev/null +++ b/tests/testsuites/queue-persist.conf @@ -0,0 +1,21 @@ +# Test for persisting messages on shutdown +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 1 +$MainMsgQueueSaveOnShutdown on +$InputTCPServerRun 13514 + +$ModLoad ../plugins/omtesting/.libs/omtesting + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$IncludeConfig work-queuemode.conf + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +$IncludeConfig work-delay.conf diff --git a/tests/testsuites/reallife.parse1 b/tests/testsuites/reallife.parse1 new file mode 100644 index 00000000..a83d2dca --- /dev/null +++ b/tests/testsuites/reallife.parse1 @@ -0,0 +1,12 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# This file contains a lot of real-life samples (of course mangled so +# that they can not be traced back to the original submitter). Note +# that IP addr 192.0.2.1 is specifically set aside for testing and +# documentation by IANA. +# rgerhards, 2009-10-19 +<29>Oct 16 20:47:24 example-p exam-pl[12345]: connect host= /192.0.2.1 +29,daemon,notice,Oct 16 20:47:24,example-p,exam-pl,exam-pl[12345]:, connect host= /192.0.2.1 diff --git a/tests/testsuites/reallife.parse2 b/tests/testsuites/reallife.parse2 new file mode 100644 index 00000000..c42f2526 --- /dev/null +++ b/tests/testsuites/reallife.parse2 @@ -0,0 +1,12 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# This file contains a lot of real-life samples (of course mangled so +# that they can not be traced back to the original submitter). Note +# that IP addr 192.0.2.1 is specifically set aside for testing and +# documentation by IANA. +# rgerhards, 2009-10-19 +<175>Oct 16 23:47:31 #001 MSWinEventLog 0#011Security#01119023582#011Fri Oct 16 16:30:44 2009#011592#011Security#011rgabcde#011User#011Success Audit#011XSXSXSN01#011Detailed Tracking#011#0112572#01119013885 +175,local5,debug,Oct 16 23:47:31,#001,#001, MSWinEventLog 0#011Security#01119023582#011Fri Oct 16 16:30:44 2009#011592#011Security#011rgabcde#011User#011Success Audit#011XSXSXSN01#011Detailed Tracking#011#0112572#01119013885 diff --git a/tests/testsuites/reallife.parse3 b/tests/testsuites/reallife.parse3 new file mode 100644 index 00000000..465635bd --- /dev/null +++ b/tests/testsuites/reallife.parse3 @@ -0,0 +1,15 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# This file contains a lot of real-life samples (of course mangled so +# that they can not be traced back to the original submitter). Note +# that IP addr 192.0.2.1 is specifically set aside for testing and +# documentation by IANA. +# rgerhards, 2009-10-19 +<175>Oct 16 2009 23:47:31 hostname tag This is a message +"2009-10-16T23:47:31+01:00", "hostname", "tag", "", "7", " This is a message" +# +<175>Oct 16 2009 23:47:31 hostname tag[1234] This is a message +"2009-10-16T23:47:31+01:00", "hostname", "tag", "1234", "7", " This is a message" diff --git a/tests/testsuites/rfctag.conf b/tests/testsuites/rfctag.conf new file mode 100644 index 00000000..8619e89e --- /dev/null +++ b/tests/testsuites/rfctag.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +# Note: the plus signs are necessary to detect truncated logs! +$template fmt,"+%syslogtag:1:32%+\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/rsf_getenv.conf b/tests/testsuites/rsf_getenv.conf new file mode 100644 index 00000000..2f2eb58c --- /dev/null +++ b/tests/testsuites/rsf_getenv.conf @@ -0,0 +1,17 @@ +# Test for RainerScript getenv() function (see .sh file for details) +# Note envvar MSGNUM must be set to "msgnum:" +# rgerhards, 2009-11-03 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +if $msg contains getenv('MSGNUM') then ?dynfile;outfmt diff --git a/tests/testsuites/rulesetmultiqueue.conf b/tests/testsuites/rulesetmultiqueue.conf new file mode 100644 index 00000000..c8a82dd6 --- /dev/null +++ b/tests/testsuites/rulesetmultiqueue.conf @@ -0,0 +1,34 @@ +# Test for multiple ruleset queues (see .sh file for details) +# rgerhards, 2009-10-30 +$IncludeConfig diag-common.conf +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 + +# general definition +$template outfmt,"%msg:F,58:2%\n" + +# create the individual rulesets +$ruleset file1 +$RulesetCreateMainQueue on +$template dynfile1,"rsyslog.out1.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile1;outfmt + +$ruleset file2 +$RulesetCreateMainQueue on +$template dynfile2,"rsyslog.out2.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile2;outfmt + +$ruleset file3 +$RulesetCreateMainQueue on +$template dynfile3,"rsyslog.out3.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile3;outfmt + +# start listeners and bind them to rulesets +$InputTCPServerBindRuleset file1 +$InputTCPServerRun 13514 + +$InputTCPServerBindRuleset file2 +$InputTCPServerRun 13515 + +$InputTCPServerBindRuleset file3 +$InputTCPServerRun 13516 diff --git a/tests/testsuites/samples.parse_invld_regex b/tests/testsuites/samples.parse_invld_regex new file mode 100644 index 00000000..9ac2c3aa --- /dev/null +++ b/tests/testsuites/samples.parse_invld_regex @@ -0,0 +1,16 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# the actual message is not important. There is an error inside the conf +# file, and all messages will trigger the same problem. +# NOTE: it is correct that the "BAD REGULAR EXPRESSION" error message is +# *NOT* run through the rest of the propety replace, in specific through +# the CSV escaper. We do not do this because it could potentially lead +# to an obfuscated error message, and thus making problems hard to find. As +# this is a real error case, there is no problem in not obeying to the +# configured format. +# rgerhards, 2010-02-08 +<175>Feb 08 2008 23:47:31 hostname tag This is a message +"2008-02-08T23:47:31+01:00", "hostname", "tag", **NO MATCH** **BAD REGULAR EXPRESSION**, "7", " This is a message" diff --git a/tests/testsuites/sndrcv_gzip_rcvr.conf b/tests/testsuites/sndrcv_gzip_rcvr.conf new file mode 100644 index 00000000..6f7ce34b --- /dev/null +++ b/tests/testsuites/sndrcv_gzip_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# then SENDER sends to this port (not tcpflood!) +$InputTCPServerRun 13515 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_gzip_sender.conf b/tests/testsuites/sndrcv_gzip_sender.conf new file mode 100644 index 00000000..c874c068 --- /dev/null +++ b/tests/testsuites/sndrcv_gzip_sender.conf @@ -0,0 +1,8 @@ +# see tcpsndrcv.sh for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +*.* @@127.0.0.1:13515 diff --git a/tests/testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf new file mode 100644 index 00000000..65659f00 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 2514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf new file mode 100644 index 00000000..29a30145 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf @@ -0,0 +1,18 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +$ModLoad ../plugins/omudpspoof/.libs/omudpspoof +$template spoofaddr,"127.0.0.1" + +#begin action definition +$ActionOMUDPSpoofSourceNameTemplate spoofaddr +$ActionOMUDPSpoofTargetHost 127.0.0.1 +$ActionOMUDPSpoofTargetPort 2514 +$ActionOMUDPSpoofSourcePortStart 514 +$ActionOMUDPSpoofSourcePortEnd 514 +*.* :omudpspoof: diff --git a/tests/testsuites/sndrcv_omudpspoof_rcvr.conf b/tests/testsuites/sndrcv_omudpspoof_rcvr.conf new file mode 100644 index 00000000..e5401811 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_omudpspoof_sender.conf b/tests/testsuites/sndrcv_omudpspoof_sender.conf new file mode 100644 index 00000000..c0d25935 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_sender.conf @@ -0,0 +1,17 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +$ModLoad ../plugins/omudpspoof/.libs/omudpspoof +$template spoofaddr,"127.0.0.1" + +#begin action definition +$ActionOMUDPSpoofSourceNameTemplate spoofaddr +$ActionOMUDPSpoofTargetHost 127.0.0.1 +$ActionOMUDPSpoofSourcePortStart 514 +$ActionOMUDPSpoofSourcePortEnd 514 +*.* :omudpspoof: diff --git a/tests/testsuites/sndrcv_rcvr.conf b/tests/testsuites/sndrcv_rcvr.conf new file mode 100644 index 00000000..6f7ce34b --- /dev/null +++ b/tests/testsuites/sndrcv_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# then SENDER sends to this port (not tcpflood!) +$InputTCPServerRun 13515 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_sender.conf b/tests/testsuites/sndrcv_sender.conf new file mode 100644 index 00000000..f3d6ba53 --- /dev/null +++ b/tests/testsuites/sndrcv_sender.conf @@ -0,0 +1,9 @@ +# see tcpsndrcv.sh for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +*.* @@127.0.0.1:13515 diff --git a/tests/testsuites/sndrcv_tls_anon_rcvr.conf b/tests/testsuites/sndrcv_tls_anon_rcvr.conf new file mode 100644 index 00000000..01143b22 --- /dev/null +++ b/tests/testsuites/sndrcv_tls_anon_rcvr.conf @@ -0,0 +1,22 @@ +# see equally-named shell file for details +# this is the config fil for the TLS server +# rgerhards, 2009-11-11 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp + +# certificates +$DefaultNetstreamDriverCAFile testsuites/x.509/ca.pem +$DefaultNetstreamDriverCertFile testsuites/x.509/client-cert.pem +$DefaultNetstreamDriverKeyFile testsuites/x.509/client-key.pem + +$DefaultNetstreamDriver gtls # use gtls netstream driver + +# then SENDER sends to this port (not tcpflood!) +$InputTCPServerStreamDriverMode 1 +$InputTCPServerStreamDriverAuthMode anon +$InputTCPServerRun 13515 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_tls_anon_sender.conf b/tests/testsuites/sndrcv_tls_anon_sender.conf new file mode 100644 index 00000000..4a944455 --- /dev/null +++ b/tests/testsuites/sndrcv_tls_anon_sender.conf @@ -0,0 +1,19 @@ +# see tcpsndrcv.sh for details +# this is the TLS client +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +# certificates +$DefaultNetstreamDriverCAFile testsuites/x.509/ca.pem +$DefaultNetstreamDriverCertFile testsuites/x.509/client-cert.pem +$DefaultNetstreamDriverKeyFile testsuites/x.509/client-key.pem + +# Note: no TLS for the listener, this is for tcpflood! +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +# set up the action +$DefaultNetstreamDriver gtls # use gtls netstream driver +$ActionSendStreamDriverMode 1 # require TLS for the connection +$ActionSendStreamDriverAuthMode anon +*.* @@127.0.0.1:13515 diff --git a/tests/testsuites/sndrcv_udp_nonstdpt_rcvr.conf b/tests/testsuites/sndrcv_udp_nonstdpt_rcvr.conf new file mode 100644 index 00000000..65659f00 --- /dev/null +++ b/tests/testsuites/sndrcv_udp_nonstdpt_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 2514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_udp_nonstdpt_sender.conf b/tests/testsuites/sndrcv_udp_nonstdpt_sender.conf new file mode 100644 index 00000000..2975f938 --- /dev/null +++ b/tests/testsuites/sndrcv_udp_nonstdpt_sender.conf @@ -0,0 +1,9 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +*.* @127.0.0.1:2514 diff --git a/tests/testsuites/sndrcv_udp_rcvr.conf b/tests/testsuites/sndrcv_udp_rcvr.conf new file mode 100644 index 00000000..e5401811 --- /dev/null +++ b/tests/testsuites/sndrcv_udp_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_udp_sender.conf b/tests/testsuites/sndrcv_udp_sender.conf new file mode 100644 index 00000000..28e913ef --- /dev/null +++ b/tests/testsuites/sndrcv_udp_sender.conf @@ -0,0 +1,9 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +*.* @127.0.0.1 diff --git a/tests/testsuites/subsecond.conf b/tests/testsuites/subsecond.conf new file mode 100644 index 00000000..58c26cc7 --- /dev/null +++ b/tests/testsuites/subsecond.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-subseconds%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tabescape_dflt.conf b/tests/testsuites/tabescape_dflt.conf new file mode 100644 index 00000000..b9d92a37 --- /dev/null +++ b/tests/testsuites/tabescape_dflt.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tabescape_off.conf b/tests/testsuites/tabescape_off.conf new file mode 100644 index 00000000..c1eca305 --- /dev/null +++ b/tests/testsuites/tabescape_off.conf @@ -0,0 +1,10 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +$EscapeControlCharacterTab off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/threadingmq.conf b/tests/testsuites/threadingmq.conf new file mode 100644 index 00000000..b98f9b5a --- /dev/null +++ b/tests/testsuites/threadingmq.conf @@ -0,0 +1,16 @@ +# Threading test, we run a tcp flood to via an +# engine instructed to use multiple threads +# rgerhards, 2009-06-26 +$IncludeConfig diag-common.conf + +$MainMsgQueueTimeoutShutdown 100000 + +$MainMsgQueueWorkerThreadMinimumMessages 10 +$MainMsgQueueWorkerThreads 5 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +# write quickly to the output file: +$OMFileFlushOnTXEnd off +$OMFileIOBufferSize 256k +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/threadingmqaq.conf b/tests/testsuites/threadingmqaq.conf new file mode 100644 index 00000000..f0d39057 --- /dev/null +++ b/tests/testsuites/threadingmqaq.conf @@ -0,0 +1,20 @@ +# Threading test, we run a tcp flood to via an +# engine instructed to use multiple threads +# rgerhards, 2009-06-26 +$IncludeConfig diag-common.conf + +$MainMsgQueueTimeoutShutdown 10000 + +$MainMsgQueueWorkerThreadMinimumMessages 10 +$MainMsgQueueWorkerThreads 5 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +# write quickly to the output file: +$OMFileFlushOnTXEnd off +$OMFileIOBufferSize 256k +# This time, also run the action queue detached +$ActionQueueWorkerThreadMinimumMessages 10 +$ActionQueueWorkerThreads 5 +$ActionQueueType LinkedList +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/ts3164.conf b/tests/testsuites/ts3164.conf new file mode 100644 index 00000000..7aa6a8ef --- /dev/null +++ b/tests/testsuites/ts3164.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-rfc3164%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/ts3339.conf b/tests/testsuites/ts3339.conf new file mode 100644 index 00000000..df8f23ac --- /dev/null +++ b/tests/testsuites/ts3339.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-rfc3339%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tsmysql.conf b/tests/testsuites/tsmysql.conf new file mode 100644 index 00000000..f97d4b0a --- /dev/null +++ b/tests/testsuites/tsmysql.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-mysql%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tspgsql.conf b/tests/testsuites/tspgsql.conf new file mode 100644 index 00000000..eb18c091 --- /dev/null +++ b/tests/testsuites/tspgsql.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-pgsql%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/upcase-date.parse1 b/tests/testsuites/upcase-date.parse1 new file mode 100644 index 00000000..2d21222a --- /dev/null +++ b/tests/testsuites/upcase-date.parse1 @@ -0,0 +1,4 @@ +<6>AUG 10 22:18:24 2009 netips-warden2-p [audit] user=[*SMS] src=192.168.11.11 iface=5 access=9 Update State Reset +6,kern,info,Aug 10 22:18:24,2009,,, netips-warden2-p [audit] user=[*SMS] src=192.168.11.11 iface=5 access=9 Update State Reset +#Example from RFC3164, section 5.4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/valid.conf b/tests/testsuites/valid.conf new file mode 100644 index 00000000..250f0546 --- /dev/null +++ b/tests/testsuites/valid.conf @@ -0,0 +1,3 @@ +# This is an invalid config file that shall trigger an exit code +# with the config verification run +*.* /tmp/data.log diff --git a/tests/testsuites/weird.parse1 b/tests/testsuites/weird.parse1 new file mode 100644 index 00000000..e8b90c74 --- /dev/null +++ b/tests/testsuites/weird.parse1 @@ -0,0 +1,37 @@ +# some really weird samples, some of them seen in practice, +# some other deliberately generated. The main point is that they +# should not cause an abort... +<14>Aug 30 23:00:05 X4711 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +14,user,info,Aug 30 23:00:05,X4711,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA, +# important: the following line has a SP at the end of the line! +<14>Aug 30 23:00:05 X4711 +14,user,info,Aug 30 23:00:05,X4711,,, +# and this one NOT +<14>Aug 30 23:00:05 X4711 +14,user,info,Aug 30 23:00:05,X4711,,, +# there is a SP at the end of the line +<14>Aug 30 23:00:05 +14,user,info,Aug 30 23:00:05,localhost,,, +# and here is no SP at the end of the line +<14>Aug 30 23:00:05 +14,user,info,Aug 30 23:00:05,localhost,,, +# unfortunately, I can not test missing dates with this test suite, because +# we would have the current date in the response, which we can not check against +# +# and now the same tests with RFC3339 data - this can make a difference +# as a different date parser is involved. +# +<14>2010-08-30T23:00:05Z X4711 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +14,user,info,Aug 30 23:00:05,X4711,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA, +# important: the following line has a SP at the end of the line! +<14>2010-08-30T23:00:05Z X4711 +14,user,info,Aug 30 23:00:05,X4711,,, +# and this one NOT +<14>2010-08-30T23:00:05Z X4711 +14,user,info,Aug 30 23:00:05,X4711,,, +# there is a SP at the end of the line +<14>2010-08-30T23:00:05Z +14,user,info,Aug 30 23:00:05,localhost,,, +# and here is no SP at the end of the line +<14>2010-08-30T23:00:05Z +14,user,info,Aug 30 23:00:05,localhost,,, diff --git a/tests/testsuites/x.509/ca-key.pem b/tests/testsuites/x.509/ca-key.pem new file mode 100644 index 00000000..1e1a0b26 --- /dev/null +++ b/tests/testsuites/x.509/ca-key.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDZnIJGJH80j2DPBXdxgmFmBRMoBnpwQb8yhRJcJacaWigRAhp4 +wdo07rR+EpuBJHD/5ImIygUwCj/XWAs4JKm3LqK2ih1gUy/s6Tg2O5t3k11kdjEH +MKUxDOLs441dEwERPQtePEoy2POzViIyy959ZJorkdnwC4LBKdQVLEELlwIDAQAB +AoGAEQWvoRoAw1VF3tvQHJZ01Pyno3ViRX63HJYROhkN6b9MrAvsky6iyYo0nzoI +ZQE7P6EaaxNWdYwPs2IlOoaPqeos1sGVDaK/JFuja/DduoXBdCy9RFWRaugDX/1U +iMtjtu29euvegP0r2RIxaIl9dapF5alNH5MLMyBl7XTB+/kCQQDiwHnW8jS1paSc +/risF6Ie5rKuUfVDG8hqMEiKyczSHwUVYushwCclshjM6E1TBFZqMz/8PbFW51pK +OzFS2s6/AkEA9a4044RL3AWe37LIU4hbz2Y+auRvPh8x4i2cWLzdok8Rc1EHDGLN +eHBoOQ3Q2nQS94cOx6HxpRztzBgiwpTRKQJADX9BgV7nbkyO0N2EppG9j7NRvXiZ +bcYwlsmK99/tNjCsf8pkjpy+d8rzGPdW6vMeJbIpQ910OeUJhdOiKvllRwJBAIw3 +rP/dVd5xZseNpj/mp1+rnxwq3EK8UyAfoAgVYvlr3y3NpRQwn8yJezJ07CqB7QFR +F+JgTyZJaH7/l3cusGECQQCM3HmkADAKxX6RwKe8X0Kj/36rjXEMNoq0ZdXOB7Qz +f5N6og4Da9y/ZO+XMo6P3XR/TYIYrMD8nuoR33X69kb1 +-----END RSA PRIVATE KEY----- diff --git a/tests/testsuites/x.509/ca.pem b/tests/testsuites/x.509/ca.pem new file mode 100644 index 00000000..a733eb86 --- /dev/null +++ b/tests/testsuites/x.509/ca.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICyzCCAjagAwIBAgIESFo2XjALBgkqhkiG9w0BAQUwezELMAkGA1UEBhMCVVMx +EDAOBgNVBAoTB1NvbWVPcmcxDzANBgNVBAsTBlNvbWVPVTESMBAGA1UEBxMJU29t +ZXdoZXJlMQswCQYDVQQIEwJDQTEoMCYGA1UEAxMfc29tZU5hbWUgKG5vdCBuZWNl +c3NhcmlseSBETlMhKTAeFw0wODA2MTkxMDM1MTJaFw0xODA2MTcxMDM1MjVaMHsx +CzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdTb21lT3JnMQ8wDQYDVQQLEwZTb21lT1Ux +EjAQBgNVBAcTCVNvbWV3aGVyZTELMAkGA1UECBMCQ0ExKDAmBgNVBAMTH3NvbWVO +YW1lIChub3QgbmVjZXNzYXJpbHkgRE5TISkwgZwwCwYJKoZIhvcNAQEBA4GMADCB +iAKBgNmcgkYkfzSPYM8Fd3GCYWYFEygGenBBvzKFElwlpxpaKBECGnjB2jTutH4S +m4EkcP/kiYjKBTAKP9dYCzgkqbcuoraKHWBTL+zpODY7m3eTXWR2MQcwpTEM4uzj +jV0TARE9C148SjLY87NWIjLL3n1kmiuR2fALgsEp1BUsQQuXAgMBAAGjYzBhMA8G +A1UdEwEB/wQFMAMBAf8wHgYDVR0RBBcwFYETc29tZW9uZUBleGFtcGxlLm5ldDAP +BgNVHQ8BAf8EBQMDBwQAMB0GA1UdDgQWBBT7/paNEKc65bcNe0NIhsj4cpl7iTAL +BgkqhkiG9w0BAQUDgYEAlv9ge8Koways837OLoZIam0s7wQCcwd9rWE05caps7BU +T4bfgab9U/e9mmrf3V/zXmtU6y8hhTXF5AcZv3/EmCVwsPRotgrJ+rHXTv5e2PO7 +/8C3K2Lhc89gF4qf4xZwlZU70RasKgCzZa5ivS2Y8pW6LUu6eqqgVw3pPJbW3TE= +-----END CERTIFICATE----- diff --git a/tests/testsuites/x.509/client-cert.pem b/tests/testsuites/x.509/client-cert.pem new file mode 100644 index 00000000..5bf39f81 --- /dev/null +++ b/tests/testsuites/x.509/client-cert.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICijCCAfWgAwIBAgIESFo7ITALBgkqhkiG9w0BAQUwezELMAkGA1UEBhMCVVMx +EDAOBgNVBAoTB1NvbWVPcmcxDzANBgNVBAsTBlNvbWVPVTESMBAGA1UEBxMJU29t +ZXdoZXJlMQswCQYDVQQIEwJDQTEoMCYGA1UEAxMfc29tZU5hbWUgKG5vdCBuZWNl +c3NhcmlseSBETlMhKTAeFw0wODA2MTkxMDU1MzJaFw0xMTAzMTYxMDU1MzlaMA0x +CzAJBgNVBAYTAlVTMIGcMAsGCSqGSIb3DQEBAQOBjAAwgYgCgYC+f6yCet2WJgmw +tgukOReI+avRHOfr2hLhIQkSzCOiNi0tNWMKmaQWw/D+y1FvLRq0wLDUyJK/36rB +67HKfscoNeClKTS8jhAs1mPjT57iyuoqK6VW/d2JoofklRCgDIZQrNfxHiOO+kN3 +ShLmkGqxkA3YyUty/JmF6PKWYIhQWQIDAQABo4GPMIGMMAwGA1UdEwEB/wQCMAAw +HQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMB0GA1UdEQQWMBSCEmNsaWVu +dC5leGFtcGxlLm5ldDAdBgNVHQ4EFgQUrDcwsuOF4RiHn0eboCplJSiUhfcwHwYD +VR0jBBgwFoAU+/6WjRCnOuW3DXtDSIbI+HKZe4kwCwYJKoZIhvcNAQEFA4GBAAAh +niy9ORW2AIb6lk/sa3iYczeYpGzxDM9bLZ1xSoIdoHM/v9gPG/WpAZ4ECHjx+Yk8 +4B/9gvaAmMi0FmcoIBQaEOe2P8tcIuzmum3N2F27F2+J4httiNDLJoseWVnXJUvS +dPyVOrKXdl5vVFpmViI5P+VzzMqbAQ6oNlMXIh6e +-----END CERTIFICATE----- diff --git a/tests/testsuites/x.509/client-key.pem b/tests/testsuites/x.509/client-key.pem new file mode 100644 index 00000000..05641213 --- /dev/null +++ b/tests/testsuites/x.509/client-key.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQC+f6yCet2WJgmwtgukOReI+avRHOfr2hLhIQkSzCOiNi0tNWMK +maQWw/D+y1FvLRq0wLDUyJK/36rB67HKfscoNeClKTS8jhAs1mPjT57iyuoqK6VW +/d2JoofklRCgDIZQrNfxHiOO+kN3ShLmkGqxkA3YyUty/JmF6PKWYIhQWQIDAQAB +AoGAVxrM+BqTIJlC/Ay5lP1QAB9di3ACserUkCFJY1F5h63rCU1sfIfVKl2s3+x6 +z3GZ0QV8tccCpv5wN1x8vqEqkbOvddM3rzpGkEC5PoyfCzuQBun1wnHK/JKjrfk5 +PvcaP60eTNjHZC7w78gOJJCzgzsEMrndtE+55diPmqGVtXMCQQDTZBy5WK8gZwMO +rRz1BKKyBeMYMfTJoJafGfxp0H8AUbTa0V2eb+el3kuzPCm3FQ6IgaHyGj2WqkAw +M0bfAfdXAkEA5rLna1t+2SCtgSd1DotndA4EsH4skBq9kFeD2/8T6Pf13zmBOq6O +4aNEOhgBE/R9/MI4XoU9MbOlkZvKvDuXzwJADdWSb6rXIza6o34+0+Yuw5nRB+dV +DtD8qoLn2wDzHtE6Fcv35YOLVHac26kHTd0J63MYZyDCgRa5Rq5EaBnX1wJAQYRF +XKPbXmZ9X9SI1dyZQMhKZKUwmqw9caSo+e1zBhKFbSOzo6q3QTVQxv7SL4ybyxCN +WaqVOmw+dR+9b7+s2QJAdNAw3r418rWKFKJJNTSqSqr1sYqiKvrQL6w2dpdpAeY4 +3VDCz/7/F9AEn3R7K3fZLQ7W6M62LSEjxxc1Y3LIpQ== +-----END RSA PRIVATE KEY----- diff --git a/tests/testsuites/x.509/machine-cert.pem b/tests/testsuites/x.509/machine-cert.pem new file mode 100644 index 00000000..fa2fd36e --- /dev/null +++ b/tests/testsuites/x.509/machine-cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC7TCCAligAwIBAgIESFo4GTALBgkqhkiG9w0BAQUwezELMAkGA1UEBhMCVVMx +EDAOBgNVBAoTB1NvbWVPcmcxDzANBgNVBAsTBlNvbWVPVTESMBAGA1UEBxMJU29t +ZXdoZXJlMQswCQYDVQQIEwJDQTEoMCYGA1UEAxMfc29tZU5hbWUgKG5vdCBuZWNl +c3NhcmlseSBETlMhKTAeFw0wODA2MTkxMDQyNTRaFw0xMTAzMTYxMDQyNTdaMG8x +CzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdTb21lT3JnMQ8wDQYDVQQLEwZTb21lT1Ux +EjAQBgNVBAcTCVNvbWV3aGVyZTELMAkGA1UECBMCQ0ExHDAaBgNVBAMTE21hY2hp +bmUuZXhhbXBsZS5uZXQwgZwwCwYJKoZIhvcNAQEBA4GMADCBiAKBgLJOW6lIHv8u +c6Ez7tiir64vI3aRuDmUACPybyWtyWqrLebzYtg+borWHj9y5di54NB5wpQhZQsQ +U2awNqanzUYeLGqbecbuxuLtsKlZ4knax+PwHOBTmIcN1SjbpII27Toe0VwHE5Vd +sygFFyorto6OeNLPrIcTFfwXQ2sVw325AgMBAAGjgZAwgY0wDAYDVR0TAQH/BAIw +ADAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwHgYDVR0RBBcwFYITbWFj +aGluZS5leGFtcGxlLm5ldDAdBgNVHQ4EFgQUDOHD29GdMfoDWwev4uDvItkLKKww +HwYDVR0jBBgwFoAU+/6WjRCnOuW3DXtDSIbI+HKZe4kwCwYJKoZIhvcNAQEFA4GB +AMt1iED7QzFL2Qk6VivoFY15S2XGF8rJTd3l00bwyLA5qLyLBGlB6z4qkYu7/7SW +5r7tet+1DezgHrj/1eU289m410wnQB8fGwcVLp6OX2PAlhNmVLcsipiN6rielAcP +aIg/VlBtoCFp/ymTLKgvh6DLKWhRUkFPqO2WtcQ3UUo+ +-----END CERTIFICATE----- diff --git a/tests/testsuites/x.509/machine-key.pem b/tests/testsuites/x.509/machine-key.pem new file mode 100644 index 00000000..808f00c9 --- /dev/null +++ b/tests/testsuites/x.509/machine-key.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQCyTlupSB7/LnOhM+7Yoq+uLyN2kbg5lAAj8m8lrclqqy3m82LY +Pm6K1h4/cuXYueDQecKUIWULEFNmsDamp81GHixqm3nG7sbi7bCpWeJJ2sfj8Bzg +U5iHDdUo26SCNu06HtFcBxOVXbMoBRcqK7aOjnjSz6yHExX8F0NrFcN9uQIDAQAB +AoGABHJs2c95Km8bpikX62I/VG5LiaD/wbvdtwfMWtm3PMhRKEHotLD169OERJvW +fK3CHCD1R+F/ViPNmLGLY2Oq/GqKjhKjg4sqAznw8TImBSgXCFho4sl38z+luP1o +TXFDgfV5HDDW1/F5kJlBIfXBLFdl4VO7E0ZnFt4FqSDRW2MCQQDRun/sBGM4i9hM +QdC+QwrdcgCScBpzbz4YXtI9TyGEqNahg8kXgIVUbzDdRmG68G2M98USzRs5DWB7 +YvYwmRoPAkEA2aUdUpFRb/n7XfsAiFLYOk96C82iCCQpJi0si34zlCAEbCRbQ6zw +gVDMCMSccnnWrVzqtxfN+rXycFTNyDFTtwJAPRwymfrNTnSxGcczo7y1NcE6GXFA +w9HuLfuzFtov0g/AOl/EAG0abHfZrSAM6gOUaDbp3YiWHhGfw1QamB6EUQJAClTb +MnsxeXZNZ2Wt3crI9uOk8IB/a5GD3osQbUK9Yg+vBg8nweuoswrJ1LS4lHqSJUKe +5bgckAUpEAoGhrVIuwJBAKIuqx/cSjF4Oa9xT6DzBRe7vAlKFq62lUV5SLfoSEgY +L5dvPBgAD0Styglny1s0Bu5FTlkxlFOMvUAD/O5hsQw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testsuites/x.509/request.pem b/tests/testsuites/x.509/request.pem new file mode 100644 index 00000000..c612325c --- /dev/null +++ b/tests/testsuites/x.509/request.pem @@ -0,0 +1,10 @@ +-----BEGIN NEW CERTIFICATE REQUEST----- +MIIBWDCBxAIBADANMQswCQYDVQQGEwJVUzCBnDALBgkqhkiG9w0BAQEDgYwAMIGI +AoGAvn+sgnrdliYJsLYLpDkXiPmr0Rzn69oS4SEJEswjojYtLTVjCpmkFsPw/stR +by0atMCw1MiSv9+qweuxyn7HKDXgpSk0vI4QLNZj40+e4srqKiulVv3diaKH5JUQ +oAyGUKzX8R4jjvpDd0oS5pBqsZAN2MlLcvyZhejylmCIUFkCAwEAAaARMA8GCSqG +SIb3DQEJBzECEwAwCwYJKoZIhvcNAQEFA4GBAA6mBaHFuRvcJVNoU7wDFcDexjvC +QLpDpFRSbKcKdNEQLBRD8ZNVOY4WBXQE2pE84//QnygQPKPCHSqUVdPPBabi5y2E +A2XvgYyKsrFbsrpKrVkPz5oQB4V7FRytQaQoBi//BSOu3dMaimLcAhfNQZCrQeu8 +SYWdJi5OPvrYGvgT +-----END NEW CERTIFICATE REQUEST----- diff --git a/tests/threadingmq.sh b/tests/threadingmq.sh new file mode 100755 index 00000000..ea5d7837 --- /dev/null +++ b/tests/threadingmq.sh @@ -0,0 +1,16 @@ +# test many concurrent tcp connections +# we send 100,000 messages in the hopes that his puts at least a little bit +# of pressure on the threading subsystem. To really prove it, we would need to +# push messages for several minutes, but that takes too long during the +# automatted tests (hint: do this manually after suspect changes). Thankfully, +# in practice many threading bugs result in an abort rather quickly and these +# should be covered by this test here. +# rgerhards, 2009-06-26 +echo \[threadingmq.sh\]: main queue concurrency +source $srcdir/diag.sh init +source $srcdir/diag.sh startup threadingmq.conf +source $srcdir/diag.sh injectmsg 0 100000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99999 +source $srcdir/diag.sh exit diff --git a/tests/threadingmqaq.sh b/tests/threadingmqaq.sh new file mode 100755 index 00000000..83f5fedc --- /dev/null +++ b/tests/threadingmqaq.sh @@ -0,0 +1,18 @@ +# test many concurrent tcp connections +# we send 100,000 messages in the hopes that his puts at least a little bit +# of pressure on the threading subsystem. To really prove it, we would need to +# push messages for several minutes, but that takes too long during the +# automatted tests (hint: do this manually after suspect changes). Thankfully, +# in practice many threading bugs result in an abort rather quickly and these +# should be covered by this test here. +# rgerhards, 2009-06-26 +echo \[threadingmqaq.sh\]: main/action queue concurrency +source $srcdir/diag.sh init +source $srcdir/diag.sh startup threadingmqaq.conf +#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 2 100000 +#source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh injectmsg 0 100000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99999 +source $srcdir/diag.sh exit diff --git a/tests/timestamp.sh b/tests/timestamp.sh new file mode 100755 index 00000000..71416c33 --- /dev/null +++ b/tests/timestamp.sh @@ -0,0 +1,13 @@ +echo \[timestamp.sh\]: various timestamp tests +source $srcdir/diag.sh init +source $srcdir/diag.sh nettester ts3164 udp +source $srcdir/diag.sh nettester ts3164 tcp +source $srcdir/diag.sh nettester ts3339 udp +source $srcdir/diag.sh nettester ts3339 tcp +source $srcdir/diag.sh nettester tsmysql udp +source $srcdir/diag.sh nettester tsmysql tcp +source $srcdir/diag.sh nettester tspgsql udp +source $srcdir/diag.sh nettester tspgsql tcp +source $srcdir/diag.sh nettester subsecond udp +source $srcdir/diag.sh nettester subsecond tcp +source $srcdir/diag.sh init diff --git a/tests/validation-run.sh b/tests/validation-run.sh new file mode 100755 index 00000000..cc29482a --- /dev/null +++ b/tests/validation-run.sh @@ -0,0 +1,40 @@ +# check if the configuration test run detects invalid config files. +# +# Part of the testbench for rsyslog. +# +# Copyright 2009 Rainer Gerhards and Adiscon GmbH. +# +# This file is part of rsyslog. +# +# Rsyslog is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rsyslog is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. +# +# A copy of the GPL can be found in the file "COPYING" in this distribution. +#set -x +echo \[validation-run.sh\]: testing configuraton validation +echo "testing a failed configuration verification run" +../tools/rsyslogd -dn -u2 -c4 -N1 -f$srcdir/testsuites/invalid.conf -M../runtime/.libs:../.libs +if [ $? -ne 1 ]; then + exit 1 +fi +echo testing a valid config verification run +../tools/rsyslogd -u2 -c4 -N1 -f$srcdir/testsuites/valid.conf -M../runtime/.libs:../.libs +if [ $? -ne 0 ]; then + exit 1 +fi +echo testing empty config file +../tools/rsyslogd -u2 -c4 -N1 -f/dev/null -M../runtime/.libs:../.libs +if [ $? -ne 1 ]; then + exit 1 +fi +echo SUCCESS: validation run tests diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh deleted file mode 100755 index 4825853a..00000000 --- a/tests/waitqueueempty.sh +++ /dev/null @@ -1,5 +0,0 @@ -# wait until main message queue is empty. This is currently done in -# a separate shell script so that we can change the implementation -# at some later point. -- rgerhards, 2009-05-25 -#echo WaitMainQueueEmpty | nc 127.0.0.1 13500 -echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker @@ -5,7 +5,7 @@ * * File begun on 2007-12-14 by RGerhards * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -29,6 +29,7 @@ #include <stdlib.h> #include <string.h> #include <signal.h> +#include <errno.h> #include <pthread.h> #include <assert.h> @@ -36,6 +37,7 @@ #include "dirty.h" #include "linkedlist.h" #include "threads.h" +#include "srUtils.h" /* linked list of currently-known threads */ static linkedList_t llThrds; @@ -44,23 +46,20 @@ static linkedList_t llThrds; /* Construct a new thread object */ -static rsRetVal thrdConstruct(thrdInfo_t **ppThis) +static rsRetVal +thrdConstruct(thrdInfo_t **ppThis) { DEFiRet; thrdInfo_t *pThis; assert(ppThis != NULL); - if((pThis = calloc(1, sizeof(thrdInfo_t))) == NULL) - return RS_RET_OUT_OF_MEMORY; - - /* OK, we got the element, now initialize members that should - * not be zero-filled. - */ - pThis->mutTermOK = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); - pthread_mutex_init (pThis->mutTermOK, NULL); - + CHKmalloc(pThis = calloc(1, sizeof(thrdInfo_t))); + pthread_mutex_init(&pThis->mutThrd, NULL); + pthread_cond_init(&pThis->condThrdTerm, NULL); *ppThis = pThis; + +finalize_it: RETiRet; } @@ -77,13 +76,54 @@ static rsRetVal thrdDestruct(thrdInfo_t *pThis) if(pThis->bIsActive == 1) { thrdTerminate(pThis); } - free(pThis->mutTermOK); + pthread_mutex_destroy(&pThis->mutThrd); + pthread_cond_destroy(&pThis->condThrdTerm); free(pThis); RETiRet; } +/* terminate a thread via the non-cancel interface + * This is a separate function as it involves a bit more of code. + * rgerhads, 2009-10-15 + */ +static inline rsRetVal +thrdTerminateNonCancel(thrdInfo_t *pThis) +{ + struct timespec tTimeout; + int ret; + DEFiRet; + assert(pThis != NULL); + + DBGPRINTF("request term via SIGTTIN for input thread 0x%x\n", (unsigned) pThis->thrdID); + pThis->bShallStop = TRUE; + do { + d_pthread_mutex_lock(&pThis->mutThrd); + pthread_kill(pThis->thrdID, SIGTTIN); + timeoutComp(&tTimeout, 10); /* a fixed 10ms timeout, do after lock (may take long!) */ + ret = d_pthread_cond_timedwait(&pThis->condThrdTerm, &pThis->mutThrd, &tTimeout); + d_pthread_mutex_unlock(&pThis->mutThrd); + if(Debug) { + if(ret == ETIMEDOUT) { + dbgprintf("input thread term: had a timeout waiting on thread termination\n"); + } else if(ret == 0) { + dbgprintf("input thread term: thread returned normally and is terminated\n"); + } else { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + dbgprintf("input thread term: cond_wait returned with error %d: %s\n", + err, errStr); + } + } + } while(pThis->bIsActive); + DBGPRINTF("non-cancel input thread termination succeeded for thread 0x%x\n", (unsigned) pThis->thrdID); + + RETiRet; +} + + /* terminate a thread gracefully. */ rsRetVal thrdTerminate(thrdInfo_t *pThis) @@ -91,9 +131,14 @@ rsRetVal thrdTerminate(thrdInfo_t *pThis) DEFiRet; assert(pThis != NULL); - pthread_cancel(pThis->thrdID); - pthread_join(pThis->thrdID, NULL); /* wait for cancel to complete */ - pThis->bIsActive = 0; + if(pThis->bNeedsCancel) { + DBGPRINTF("request term via canceling for input thread 0x%x\n", (unsigned) pThis->thrdID); + pthread_cancel(pThis->thrdID); + pThis->bIsActive = 0; + } else { + thrdTerminateNonCancel(pThis); + } + pthread_join(pThis->thrdID, NULL); /* wait for input thread to complete */ /* call cleanup function, if any */ if(pThis->pAfterRun != NULL) @@ -132,6 +177,11 @@ static void* thrdStarter(void *arg) sigfillset(&sigSet); pthread_sigmask(SIG_BLOCK, &sigSet, NULL); + /* but ignore SIGTTN, which we (ab)use to signal the thread to shutdown -- rgerhards, 2009-07-20 */ + sigemptyset(&sigSet); + sigaddset(&sigSet, SIGTTIN); + pthread_sigmask(SIG_UNBLOCK, &sigSet, NULL); + /* setup complete, we are now ready to execute the user code. We will not * regain control until the user code is finished, in which case we terminate * the thread. @@ -139,6 +189,16 @@ static void* thrdStarter(void *arg) iRet = pThis->pUsrThrdMain(pThis); dbgprintf("thrdStarter: usrThrdMain 0x%lx returned with iRet %d, exiting now.\n", (unsigned long) pThis->thrdID, iRet); + + /* signal master control that we exit (we do the mutex lock mostly to + * keep the thread debugger happer, it would not really be necessary with + * the logic we employ...) + */ + pThis->bIsActive = 0; + d_pthread_mutex_lock(&pThis->mutThrd); + pthread_cond_signal(&pThis->condThrdTerm); + d_pthread_mutex_unlock(&pThis->mutThrd); + ENDfunc pthread_exit(0); } @@ -147,7 +207,7 @@ static void* thrdStarter(void *arg) * executing threads. It is added at the end of the list. * rgerhards, 2007-12-14 */ -rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *)) +rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), sbool bNeedsCancel) { DEFiRet; thrdInfo_t *pThis; @@ -159,6 +219,7 @@ rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdI pThis->bIsActive = 1; pThis->pUsrThrdMain = thrdMain; pThis->pAfterRun = afterRun; + pThis->bNeedsCancel = bNeedsCancel; i = pthread_create(&pThis->thrdID, NULL, thrdStarter, pThis); CHKiRet(llAppend(&llThrds, NULL, pThis)); @@ -184,37 +245,10 @@ rsRetVal thrdInit(void) rsRetVal thrdExit(void) { DEFiRet; - iRet = llDestroy(&llThrds); - - RETiRet; -} - - -/* thrdSleep() - a fairly portable way to put a thread to sleep. It - * will wake up when - * a) the wake-time is over - * b) the thread shall be terminated - * Returns RS_RET_OK if all went well, RS_RET_TERMINATE_NOW if the calling - * thread shall be terminated and any other state if an error happened. - * rgerhards, 2007-12-17 - */ -rsRetVal -thrdSleep(thrdInfo_t *pThis, int iSeconds, int iuSeconds) -{ - DEFiRet; - struct timeval tvSelectTimeout; - - assert(pThis != NULL); - tvSelectTimeout.tv_sec = iSeconds; - tvSelectTimeout.tv_usec = iuSeconds; /* micro seconds */ - select(1, NULL, NULL, NULL, &tvSelectTimeout); - if(pThis->bShallStop) - iRet = RS_RET_TERMINATE_NOW; RETiRet; } -/* - * vi:set ai: +/* vi:set ai: */ @@ -25,12 +25,14 @@ /* the thread object */ struct thrdInfo { - pthread_mutex_t *mutTermOK; /* Is it ok to terminate that thread now? */ + pthread_mutex_t mutThrd;/* mutex for handling long-running operations and shutdown */ + pthread_cond_t condThrdTerm;/* condition: thread terminates (used just for shutdown loop) */ int bIsActive; /* Is thread running? */ int bShallStop; /* set to 1 if the thread should be stopped ? */ rsRetVal (*pUsrThrdMain)(struct thrdInfo*); /* user thread main to be called in new thread */ rsRetVal (*pAfterRun)(struct thrdInfo*); /* cleanup function */ pthread_t thrdID; + sbool bNeedsCancel; /* must input be terminated by pthread_cancel()? */ }; /* prototypes */ @@ -38,8 +40,7 @@ rsRetVal thrdExit(void); rsRetVal thrdInit(void); rsRetVal thrdTerminate(thrdInfo_t *pThis); rsRetVal thrdTerminateAll(void); -rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *)); -rsRetVal thrdSleep(thrdInfo_t *pThis, int iSeconds, int iuSeconds); +rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), sbool); /* macros (replace inline functions) */ diff --git a/tools/Makefile.am b/tools/Makefile.am index e523b854..f0a8df5f 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -13,8 +13,14 @@ rsyslogd_SOURCES = \ omfwd.h \ omfile.c \ omfile.h \ + ompipe.c \ + ompipe.h \ omdiscard.c \ omdiscard.h \ + pmrfc5424.c \ + pmrfc5424.h \ + pmrfc3164.c \ + pmrfc3164.h \ iminternal.c \ iminternal.h \ pidfile.c \ @@ -26,8 +32,10 @@ rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS -sbin_PROGRAMS += rsyslog_diag_hostname msggen +sbin_PROGRAMS += rsyslog_diag_hostname msggen zpipe rsyslog_diag_hostname_SOURCES = gethostn.c +zpipe_SOURCES = zpipe.c +zpipe_LDADD = -lz msggen_SOURCES = msggen.c endif diff --git a/tools/iminternal.c b/tools/iminternal.c index 0ceff3d8..bd1fa128 100644 --- a/tools/iminternal.c +++ b/tools/iminternal.c @@ -89,7 +89,7 @@ finalize_it: * The interface of this function is modelled after syslogd/logmsg(), * for which it is an "replacement". */ -rsRetVal iminternalAddMsg(int pri, msg_t *pMsg, int flags) +rsRetVal iminternalAddMsg(int pri, msg_t *pMsg) { DEFiRet; iminternal_t *pThis; @@ -100,7 +100,6 @@ rsRetVal iminternalAddMsg(int pri, msg_t *pMsg, int flags) pThis->pri = pri; pThis->pMsg = pMsg; - pThis->flags = flags; CHKiRet(llAppend(&llMsgs, NULL, (void*) pThis)); @@ -119,7 +118,7 @@ finalize_it: * from the list and return it to the caller. The caller is * responsible for freeing the message! */ -rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg, int *pFlags) +rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg) { DEFiRet; iminternal_t *pThis; @@ -127,11 +126,9 @@ rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg, int *pFlags) assert(pPri != NULL); assert(ppMsg != NULL); - assert(pFlags != NULL); CHKiRet(llGetNextElt(&llMsgs, &llCookie, (void*)&pThis)); *pPri = pThis->pri; - *pFlags = pThis->flags; *ppMsg = pThis->pMsg; pThis->pMsg = NULL; /* we do no longer own it - important for destructor */ diff --git a/tools/iminternal.h b/tools/iminternal.h index 8dc0f171..f1062a15 100644 --- a/tools/iminternal.h +++ b/tools/iminternal.h @@ -35,15 +35,14 @@ struct iminternal_s { /* config file sysline parse entry */ int pri; msg_t *pMsg; /* the message (in all its glory) */ - int flags; }; typedef struct iminternal_s iminternal_t; /* prototypes */ rsRetVal modInitIminternal(void); rsRetVal modExitIminternal(void); -rsRetVal iminternalAddMsg(int pri, msg_t *pMsg, int flags); +rsRetVal iminternalAddMsg(int pri, msg_t *pMsg); rsRetVal iminternalHaveMsgReady(int* pbHaveOne); -rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg, int *pFlags); +rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg); #endif /* #ifndef IMINTERNAL_H_INCLUDED */ diff --git a/tools/omfile.c b/tools/omfile.c index c7283e4d..424dd70f 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -1,18 +1,22 @@ /* omfile.c * This is the implementation of the build-in file output module. * - * Handles: eTypeCONSOLE, eTypeTTY, eTypeFILE, eTypePIPE - * * NOTE: read comments in module-template.h to understand how this file * works! * * File begun on 2007-07-21 by RGerhards (extracted from syslogd.c) - * This file is under development and has not yet arrived at being fully - * self-contained and a real object. So far, it is mostly an excerpt - * of the "old" message code without any modifications. However, it - * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * A large re-write of this file was done in June, 2009. The focus was + * to introduce many more features (like zipped writing), clean up the code + * and make it more reliable. In short, that rewrite tries to provide a new + * solid basis for the next three to five years to come. During it, bugs + * may have been introduced ;) -- rgerhards, 2009-06-04 + * + * Note that as of 2010-02-28 this module does no longer handle + * pipes. These have been moved to ompipe, to reduced the entanglement + * between the two different functionalities. -- rgerhards + * + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -41,6 +45,7 @@ #include <assert.h> #include <errno.h> #include <ctype.h> +#include <libgen.h> #include <unistd.h> #include <sys/file.h> @@ -48,7 +53,7 @@ # include <fcntl.h> #endif -#include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" @@ -57,6 +62,9 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "stream.h" +#include "unicode-helper.h" +#include "atomic.h" MODULE_TYPE_OUTPUT @@ -64,46 +72,74 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +DEFobjCurrIf(strm) + +/* for our current LRU mechanism, we need a monotonically increasing counters. We use + * it much like a "Lamport logical clock": we do not need the actual time, we just need + * to know the sequence in which files were accessed. So we use a simple counter to + * create that sequence. We use an unsigned 64 bit value which is extremely unlike to + * wrap within the lifetime of a process. If we process 1,000,000 file writes per + * second, the process could still exist over 500,000 years before a wrap to 0 happens. + * That should be sufficient (and even than, there would no really bad effect ;)). + * The variable below is the global counter/clock. + */ +#if HAVE_ATOMIC_BUILTINS_64BIT +static uint64 clockFileAccess = 0; +#else +static unsigned clockFileAccess = 0; +#endif +/* and the "tick" function */ +static inline uint64 +getClockFileAccess(void) +{ + return ATOMIC_INC_AND_FETCH(clockFileAccess); +} + /* The following structure is a dynafile name cache entry. */ struct s_dynaFileCacheEntry { - uchar *pName; /* name currently open, if dynamic name */ - short fd; /* name associated with file name in cache */ - time_t lastUsed; /* for LRU - last access */ + uchar *pName; /* name currently open, if dynamic name */ + strm_t *pStrm; /* our output stream */ + uint64 clkTickAccessed;/* for LRU - based on clockFileAccess */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; +#define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ +#define FLUSH_INTRVL_DFLT 1 /* default buffer flush interval (in seconds) */ + +#define DFLT_bForceChown 0 /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ static int fCreateMode = 0644; /* mode to use when creating files */ static int fDirCreateMode = 0700; /* mode to use when creating files */ static int bFailOnChown; /* fail if chown fails? */ +static int bForceChown = DFLT_bForceChown; /* Force chown() on existing files? */ static uid_t fileUID; /* UID to be used for newly created files */ static uid_t fileGID; /* GID to be used for newly created files */ static uid_t dirUID; /* UID to be used for newly created directories */ static uid_t dirGID; /* GID to be used for newly created directories */ -static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ +static int bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ -static uchar *pszTplName = NULL; /* name of the default template to use */ +static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ +static sbool bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */ +static int64 iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ +static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ +uchar *pszFileDfltTplName = NULL; /* name of the default template to use */ /* end globals for default values */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ - short fd; /* file descriptor for (current) file */ - enum { - eTypeFILE, - eTypeTTY, - eTypeCONSOLE, - eTypePIPE - } fileType; + strm_t *pStrm; /* our output stream */ + strmType_t strmType; /* stream type, used for named pipes */ char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ + sbool bForceChown; /* force chown() on existing files? */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -117,8 +153,12 @@ typedef struct _instanceData { * pointer points to the overall structure. */ dynaFileCacheEntry **dynCache; - off_t f_sizeLimit; /* file size limit, 0 = no limit */ - char *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ + int iZipLevel; /* zip mode to use for this selector */ + int iIOBufSize; /* size of associated io buffer */ + int iFlushInterval; /* how fast flush buffer on inactivity? */ + sbool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; @@ -132,23 +172,27 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - printf("[dynamic]\n\ttemplate='%s'" + dbgprintf("[dynamic]\n\ttemplate='%s'" "\tfile cache size=%d\n" "\tcreate directories: %s\n" "\tfile owner %d, group %d\n" "\tdirectory owner %d, group %d\n" + "\tforce chown() for all files: %s\n" + "\tdir create mode 0%3.3o, file create mode 0%3.3o\n" "\tfail if owner/group can not be set: %s\n", pData->f_fname, pData->iDynaFileCacheSize, pData->bCreateDirs ? "yes" : "no", pData->fileUID, pData->fileGID, pData->dirUID, pData->dirGID, + pData->bForceChown ? "yes" : "no", + pData->fDirCreateMode, pData->fCreateMode, pData->bFailOnChown ? "yes" : "no" ); } else { /* regular file */ - printf("%s", pData->f_fname); - if (pData->fd == -1) - printf(" (unused)"); + dbgprintf("%s", pData->f_fname); + if (pData->pStrm == NULL) + dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -168,13 +212,13 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal) errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; iNewVal = 1; - } else if(iNewVal > 10000) { + } else if(iNewVal > 1000) { snprintf((char*) errMsg, sizeof(errMsg)/sizeof(uchar), - "DynaFileCacheSize maximum is 10,000 (%d given), changed to 10,000.", iNewVal); + "DynaFileCacheSize maximum is 1,000 (%d given), changed to 1,000.", iNewVal); errno = 0; errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; - iNewVal = 10000; + iNewVal = 1000; } iDynaFileCacheSize = iNewVal; @@ -199,14 +243,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR struct outchannel *pOch; char szBuf[128]; /* should be more than sufficient */ - /* this must always be a file, because we can not set a size limit - * on a pipe... - * rgerhards 2005-06-21: later, this will be a separate type, but let's - * emulate things for the time being. When everything runs, we can - * extend it... - */ - pData->fileType = eTypeFILE; - ++p; /* skip '$' */ i = 0; /* get outchannel name */ @@ -241,79 +277,21 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR } /* OK, we finally got a correct template. So let's use it... */ - strncpy((char*) pData->f_fname, (char*) pOch->pszFileTemplate, MAXFNAME); - pData->f_sizeLimit = pOch->uSizeLimit; + ustrncpy(pData->f_fname, pOch->pszFileTemplate, MAXFNAME); + pData->iSizeLimit = pOch->uSizeLimit; /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit; + pData->pszSizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); + (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName); finalize_it: RETiRet; } -/* rgerhards 2005-06-21: Try to resolve a size limit - * situation. This first runs the command, and then - * checks if we are still above the treshold. - * returns 0 if ok, 1 otherwise - * TODO: consider moving the initial check in here, too - */ -int resolveFileSizeLimit(instanceData *pData) -{ - uchar *pParams; - uchar *pCmd; - uchar *p; - off_t actualFileSize; - ASSERT(pData != NULL); - - if(pData->f_sizeLimitCmd == NULL) - return 1; /* nothing we can do in this case... */ - - /* the execProg() below is probably not great, but at least is is - * fairly secure now. Once we change the way file size limits are - * handled, we should also revisit how this command is run (and - * with which parameters). rgerhards, 2007-07-20 - */ - /* we first check if we have command line parameters. We assume this, - * when we have a space in the program name. If we find it, everything after - * the space is treated as a single argument. - */ - if((pCmd = (uchar*)strdup((char*)pData->f_sizeLimitCmd)) == NULL) { - /* there is not much we can do - we make syslogd close the file in this case */ - return 1; - } - - for(p = pCmd ; *p && *p != ' ' ; ++p) { - /* JUST SKIP */ - } - - if(*p == ' ') { - *p = '\0'; /* pretend string-end */ - pParams = p+1; - } else - pParams = NULL; - - execProg(pCmd, 1, pParams); - - free(pCmd); - - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - - actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { - /* OK, it didn't work out... */ - return 1; - } - - return 0; -} - - /* This function deletes an entry from the dynamic file name * cache. A pointer to the cache must be passed in as well * as the index of the to-be-deleted entry. This index may @@ -321,23 +299,24 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) +static rsRetVal +dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { + DEFiRet; ASSERT(pCache != NULL); - BEGINfunc; - if(pCache[iEntry] == NULL) FINALIZE; DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, - pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName); + pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName); /* if the name is NULL, this is an improperly initilized entry which * needs to be discarded. In this case, neither the file is to be closed * not the name to be freed. */ if(pCache[iEntry]->pName != NULL) { - close(pCache[iEntry]->fd); + if(pCache[iEntry]->pStrm != NULL) + strm.Destruct(&pCache[iEntry]->pStrm); d_free(pCache[iEntry]->pName); pCache[iEntry]->pName = NULL; } @@ -348,7 +327,7 @@ static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int b } finalize_it: - ENDfunc; + RETiRet; } @@ -356,7 +335,8 @@ finalize_it: * relevant files. Part of Shutdown and HUP processing. * rgerhards, 2008-10-23 */ -static inline void dynaFileFreeCacheEntries(instanceData *pData) +static inline void +dynaFileFreeCacheEntries(instanceData *pData) { register int i; ASSERT(pData != NULL); @@ -392,25 +372,33 @@ static void dynaFileFreeCache(instanceData *pData) static rsRetVal prepareFile(instanceData *pData, uchar *newFileName) { + int fd; DEFiRet; - if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); - FINALIZE; /* we are done in this case */ - } if(access((char*)newFileName, F_OK) == 0) { - /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); + if(pData->bForceChown) { + /* Try to fix wrong ownership set by someone else. Note that this code + * will no longer work once we have made the $PrivDrop code fully secure. + * This change is based on an idea of Michael Terry, provided as part of + * the effort to make rsyslogd the Ubuntu default syslogd. + * rgerhards, 2009-09-11 + */ + if(chown((char*)newFileName, pData->fileUID, pData->fileGID) != 0) { + if(pData->bFailOnChown) { + int eSave = errno; + errno = eSave; + } + } + } } else { - pData->fd = -1; /* file does not exist, create it (and eventually parent directories */ + fd = -1; if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code * fall through to error handler below. */ - if(makeFileParentDirs(newFileName, strlen((char*)newFileName), + if(makeFileParentDirs(newFileName, ustrlen(newFileName), pData->fDirCreateMode, pData->dirUID, pData->dirGID, pData->bFailOnChown) != 0) { ABORT_FINALIZE(RS_RET_ERR); /* we give up */ @@ -419,17 +407,17 @@ prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, + fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); - if(pData->fd != -1) { + if(fd != -1) { /* check and set uid/gid */ - if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { + if(pData->bForceChown || pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { + if(fchown(fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; - close(pData->fd); - pData->fd = -1; + close(fd); + fd = -1; errno = eSave; } /* we will silently ignore the chown() failure @@ -437,19 +425,42 @@ prepareFile(instanceData *pData, uchar *newFileName) */ } } + close(fd); /* close again, as we need a stream further on */ } } -finalize_it: - /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean - * non-open file descriptor. Anyhow, I leave this comment for the time being to that if - * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 - */ - if(pData->fd != -1 && isatty(pData->fd)) { - DBGPRINTF("file %d is a tty file\n", pData->fd); - pData->fileType = eTypeTTY; - untty(); - } + /* the copies below are clumpsy, but there is no way around given the + * anomalies in dirname() and basename() [they MODIFY the provided buffer...] + */ + uchar szNameBuf[MAXFNAME]; + uchar szDirName[MAXFNAME]; + uchar szBaseName[MAXFNAME]; + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szDirName, (uchar*)dirname((char*)szNameBuf), MAXFNAME); + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szBaseName, (uchar*)basename((char*)szNameBuf), MAXFNAME); + + CHKiRet(strm.Construct(&pData->pStrm)); + CHKiRet(strm.SetFName(pData->pStrm, szBaseName, ustrlen(szBaseName))); + CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName))); + CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); + CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); + CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); + CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); + CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); + CHKiRet(strm.SetsType(pData->pStrm, pData->strmType)); + CHKiRet(strm.SetiSizeLimit(pData->pStrm, pData->iSizeLimit)); + /* set the flush interval only if we actually use it - otherwise it will activate + * async processing, which is a real performance waste if we do not do buffered + * writes! -- rgerhards, 2009-07-06 + */ + if(!pData->bFlushOnTXEnd) + CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval)); + if(pData->pszSizeLimitCmd != NULL) + CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd))); + CHKiRet(strm.ConstructFinalize(pData->pStrm)); + +finalize_it: RETiRet; } @@ -462,15 +473,16 @@ finalize_it: * be written. * This is a helper to writeFile(). rgerhards, 2007-07-03 */ -static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) +static inline rsRetVal +prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) { - time_t ttOldest; /* timestamp of oldest element */ + uint64 ctOldest; /* "timestamp" of oldest element */ int iOldest; int i; int iFirstFree; + rsRetVal localRet; dynaFileCacheEntry **pCache; - - BEGINfunc + DEFiRet; ASSERT(pData != NULL); ASSERT(newFileName != NULL); @@ -481,33 +493,35 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg * I *hope* this will be a performance enhancement. */ if( (pData->iCurrElt != -1) - && !strcmp((char*) newFileName, (char*) pCache[pData->iCurrElt]->pName)) { + && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ - pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + pCache[pData->iCurrElt]->clkTickAccessed = getClockFileAccess(); + // LRU needs only a strictly monotonically increasing counter, so such a one could do + FINALIZE; } /* ok, no luck. Now let's search the table if we find a matching spot. * While doing so, we also prepare for creation of a new one. */ + pData->iCurrElt = -1; /* invalid current element pointer */ iFirstFree = -1; /* not yet found */ iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */ - ttOldest = time(NULL) + 1; /* there must always be an older one */ + ctOldest = getClockFileAccess(); /* there must always be an older one */ for(i = 0 ; i < pData->iCurrCacheSize ; ++i) { if(pCache[i] == NULL) { if(iFirstFree == -1) iFirstFree = i; } else { /* got an element, let's see if it matches */ - if(!strcmp((char*) newFileName, (char*) pCache[i]->pName)) { + if(!ustrcmp(newFileName, pCache[i]->pName)) { /* we found our element! */ - pData->fd = pCache[i]->fd; + pData->pStrm = pCache[i]->pStrm; pData->iCurrElt = i; - pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + pCache[i]->clkTickAccessed = getClockFileAccess(); /* update "timestamp" for LRU */ + FINALIZE; } /* did not find it - so lets keep track of the counters for LRU */ - if(pCache[i]->lastUsed < ttOldest) { - ttOldest = pCache[i]->lastUsed; + if(pCache[i]->clkTickAccessed < ctOldest) { + ctOldest = pCache[i]->clkTickAccessed; iOldest = i; } } @@ -524,18 +538,14 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg iFirstFree = iOldest; /* this one *is* now free ;) */ } else { /* we need to allocate memory for the cache structure */ - pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)); - if(pCache[iFirstFree] == NULL) { - DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n"); - return -1; - } + CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry))); } /* Ok, we finally can open the file */ - prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ + localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ /* file is either open now or an error state set */ - if(pData->fd == -1) { + if(pData->pStrm == NULL) { /* do not report anything if the message is an internally-generated * message. Otherwise, we could run into a never-ending loop. The bad * news is that we also lose errors on startup messages, but so it is. @@ -543,22 +553,43 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if(iMsgOpts & INTERNAL_MSG) { DBGPRINTF("Could not open dynaFile, discarding message\n"); } else { - errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName); + errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } dynaFileDelCacheEntry(pCache, iFirstFree, 1); - pData->iCurrElt = -1; - return -1; + ABORT_FINALIZE(localRet); } - pCache[iFirstFree]->fd = pData->fd; - pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */ - pCache[iFirstFree]->lastUsed = time(NULL); + CHKmalloc(pCache[iFirstFree]->pName = ustrdup(newFileName)); + pCache[iFirstFree]->pStrm = pData->pStrm; + pCache[iFirstFree]->clkTickAccessed = getClockFileAccess(); pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); - ENDfunc +finalize_it: + RETiRet; +} + - return 0; +/* do the actual write process. This function is to be called once we are ready for writing. + * It will do buffered writes and persist data only when the buffer is full. Note that we must + * be careful to detect when the file handle changed. + * rgerhards, 2009-06-03 + */ +static rsRetVal +doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) +{ + DEFiRet; + ASSERT(pData != NULL); + ASSERT(pszBuf != NULL); + +dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); + if(pData->pStrm != NULL){ + CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); + FINALIZE; + } + +finalize_it: + RETiRet; } @@ -566,10 +597,9 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg * will be called for all outputs using file semantics, * for example also for pipes. */ -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal +writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) { - off_t actualFileSize; - int iLenWritten; DEFiRet; ASSERT(pData != NULL); @@ -578,106 +608,30 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa * check if it still is ok or a new file needs to be created */ if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - /* create the message based on format specified */ -again: - /* check if we have a file size limit and, if so, - * obey to it. - */ - if(pData->f_sizeLimit != 0) { - actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { - char errMsg[256]; - /* for now, we simply disable a file once it is - * beyond the maximum size. This is better than having - * us aborted by the OS... rgerhards 2005-06-21 - */ - (void) close(pData->fd); - /* try to resolve the situation */ - if(resolveFileSizeLimit(pData) != 0) { - /* didn't work out, so disable... */ - snprintf(errMsg, sizeof(errMsg), - "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg); - ABORT_FINALIZE(RS_RET_DISABLE_ACTION); - } else { - snprintf(errMsg, sizeof(errMsg), - "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); - } + CHKiRet(prepareDynFile(pData, ppString[1], iMsgOpts)); + } else { /* "regular", non-dynafile */ + if(pData->pStrm == NULL) { + CHKiRet(prepareFile(pData, pData->f_fname)); } } - iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); -//dbgprintf("lenwritten: %d\n", iLenWritten); - if(iLenWritten < 0) { - int e = errno; - char errStr[1024]; - rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); - - /* If a named pipe is full, we suspend this action for a while */ - if(pData->fileType == eTypePIPE && e == EAGAIN) - ABORT_FINALIZE(RS_RET_SUSPENDED); - - close(pData->fd); - pData->fd = -1; /* tell that fd is no longer open! */ - if(pData->bDynamicName && pData->iCurrElt != -1) { - /* in this case, we need to invalidate the name in the cache, too - * otherwise, an invalid fd may show up if we had a file name change. - * rgerhards, 2009-03-19 - */ - pData->dynCache[pData->iCurrElt]->fd = -1; - } - /* Check for EBADF on TTY's due to vhangup() - * Linux uses EIO instead (mrn 12 May 96) - */ - if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) -#ifdef linux - && e == EIO -#else - && e == EBADF -#endif - ) { - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC); - if (pData->fd < 0) { - iRet = RS_RET_SUSPENDED; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } else { - untty(); - goto again; - } - } else { - iRet = RS_RET_SUSPENDED; - errno = e; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } - } else if (pData->bSyncFile) { - fsync(pData->fd); - } + CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); finalize_it: + if(iRet != RS_RET_OK) { + /* in v5, we shall return different states for message-caused failure (but only there!) */ + if(pData->strmType == STREAMTYPE_NAMED_PIPE) + iRet = RS_RET_DISABLE_ACTION; /* this is the traditional semantic -- rgerhards, 2010-01-15 */ + else + iRet = RS_RET_SUSPENDED; + } RETiRet; } BEGINcreateInstance CODESTARTcreateInstance - pData->fd = -1; + pData->pStrm = NULL; ENDcreateInstance @@ -685,8 +639,8 @@ BEGINfreeInstance CODESTARTfreeInstance if(pData->bDynamicName) { dynaFileFreeCache(pData); - } else if(pData->fd != -1) - close(pData->fd); + } else if(pData->pStrm != NULL) + strm.Destruct(&pData->pStrm); ENDfreeInstance @@ -694,41 +648,50 @@ BEGINtryResume CODESTARTtryResume ENDtryResume +BEGINbeginTransaction +CODESTARTbeginTransaction + /* we have nothing to do to begin a transaction */ +ENDbeginTransaction + + +BEGINendTransaction +CODESTARTendTransaction + if(pData->bFlushOnTXEnd) { + CHKiRet(strm.Flush(pData->pStrm)); + } +finalize_it: +ENDendTransaction + + BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->f_fname); - iRet = writeFile(ppString, iMsgOpts, pData); + DBGPRINTF("file to log to: %s\n", pData->f_fname); + CHKiRet(writeFile(ppString, iMsgOpts, pData)); + if(!bCoreSupportsBatching && pData->bFlushOnTXEnd) { + CHKiRet(strm.Flush(pData->pStrm)); + } +finalize_it: + if(iRet == RS_RET_OK) + iRet = RS_RET_DEFER_COMMIT; ENDdoAction BEGINparseSelectorAct CODESTARTparseSelectorAct - /* yes, the if below is redundant, but I need it now. Will go away as - * the code further changes. -- rgerhards, 2007-07-25 - */ - if(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-') { - if((iRet = createInstance(&pData)) != RS_RET_OK) { - ENDfunc - return iRet; /* this can not use RET_iRet! */ - } - } else { - /* this is not clean, but we need it for the time being - * TODO: remove when cleaning up modularization - */ - ENDfunc - return RS_RET_CONFLINE_UNPROCESSED; - } + if(!(*p == '$' || *p == '?' || *p == '/' || *p == '-')) + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + + CHKiRet(createInstance(&pData)); if(*p == '-') { pData->bSyncFile = 0; p++; } else { - pData->bSyncFile = bEnableSync ? 1 : 0; + pData->bSyncFile = bEnableSync; } - pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */ + pData->iSizeLimit = 0; /* default value, use outchannels to configure! */ - switch (*p) - { + switch(*p) { case '$': CODE_STD_STRING_REQUESTparseSelectorAct(1) /* rgerhards 2005-06-21: this is a special setting for output-channel @@ -738,13 +701,8 @@ CODESTARTparseSelectorAct * rgerhards, 2007-07-24: output-channels will go away. We keep them * for compatibility reasons, but seems to have been a bad idea. */ - if((iRet = cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)) == RS_RET_OK) { - pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - } + CHKiRet(cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)); + pData->bDynamicName = 0; break; case '?': /* This is much like a regular file handle, but we need to obtain @@ -752,84 +710,70 @@ CODESTARTparseSelectorAct */ CODE_STD_STRING_REQUESTparseSelectorAct(2) ++p; /* eat '?' */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); /* "filename" is actually a template name, we need this as string 1. So let's add it * to the pOMSR. -- rgerhards, 2007-07-27 */ - if((iRet = OMSRsetEntry(*ppOMSR, 1, (uchar*)strdup((char*) pData->f_fname), OMSR_NO_RQD_TPL_OPTS)) != RS_RET_OK) - break; + CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(pData->f_fname), OMSR_NO_RQD_TPL_OPTS)); pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ - pData->fCreateMode = fCreateMode; /* freeze current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */ - /* we now allocate the cache table. We use calloc() intentionally, as we - * need all pointers to be initialized to NULL pointers. - */ - if((pData->dynCache = (dynaFileCacheEntry**) - calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n"); - } + /* we now allocate the cache table */ + CHKmalloc(pData->dynCache = (dynaFileCacheEntry**) + calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))); break; - case '|': + /* case '|': while pipe support has been removed, I leave the code in in case we + * need high-performance pipes at a later stage (unlikely). -- rgerhards, 2010-02-28 + */ case '/': CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* rgerhards, 2007-0726: first check if file or pipe */ + /* we now have *almost* the same semantics for files and pipes, but we still need + * to know we deal with a pipe, because we must do non-blocking opens in that case + * (to keep consistent with traditional semantics and prevent rsyslog from hanging). + */ if(*p == '|') { - pData->fileType = eTypePIPE; ++p; + pData->strmType = STREAMTYPE_NAMED_PIPE; } else { - pData->fileType = eTypeFILE; + pData->strmType = STREAMTYPE_FILE_SINGLE; } - /* rgerhards 2004-11-17: from now, we need to have different - * processing, because after the first comma, the template name - * to use is specified. So we need to scan for the first coma first - * and then look at the rest of the line. - */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - - /* at this stage, we ignore the return value of prepareFile, this is taken - * care of in later steps. -- rgerhards, 2009-03-19 + break; + default: + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* freeze current paremeters for this action */ + pData->iDynaFileCacheSize = iDynaFileCacheSize; + pData->fCreateMode = fCreateMode; + pData->fDirCreateMode = fDirCreateMode; + pData->bCreateDirs = bCreateDirs; + pData->bFailOnChown = bFailOnChown; + pData->bForceChown = bForceChown; + pData->fileUID = fileUID; + pData->fileGID = fileGID; + pData->dirUID = dirUID; + pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; + pData->bFlushOnTXEnd = bFlushOnTXEnd; + pData->iIOBufSize = (int) iIOBufSize; + pData->iFlushInterval = iFlushInterval; + + if(pData->bDynamicName == 0) { + /* try open and emit error message if not possible. At this stage, we ignore the + * return value of prepareFile, this is taken care of in later steps. */ prepareFile(pData, pData->f_fname); - if(pData->fd < 0 ) { - pData->fd = -1; + if(pData->pStrm == NULL) { DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); - break; } - if(strcmp((char*) p, _PATH_CONSOLE) == 0) - pData->fileType = eTypeCONSOLE; - break; - default: - iRet = RS_RET_CONFLINE_UNPROCESSED; - break; } CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -845,14 +789,19 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a dirUID = -1; dirGID = -1; bFailOnChown = 1; + bForceChown = DFLT_bForceChown; iDynaFileCacheSize = 10; fCreateMode = 0644; fDirCreateMode = 0700; bCreateDirs = 1; bEnableSync = 0; - if(pszTplName != NULL) { - free(pszTplName); - pszTplName = NULL; + iZipLevel = 0; + bFlushOnTXEnd = 1; + iIOBufSize = IOBUF_DFLT_SIZE; + iFlushInterval = FLUSH_INTRVL_DFLT; + if(pszFileDfltTplName != NULL) { + free(pszFileDfltTplName); + pszFileDfltTplName = NULL; } return RS_RET_OK; @@ -865,9 +814,9 @@ CODESTARTdoHUP dynaFileFreeCacheEntries(pData); pData->iCurrElt = -1; /* invalidate current element */ } else { - if(pData->fd != -1) { - close(pData->fd); - pData->fd = -1; + if(pData->pStrm != NULL) { + strm.Destruct(&pData->pStrm); + pData->pStrm = NULL; } } ENDdoHUP @@ -875,14 +824,16 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit - if(pszTplName != NULL) - free(pszTplName); + objRelease(errmsg, CORE_COMPONENT); + objRelease(strm, CORE_COMPONENT); + free(pszFileDfltTplName); ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ CODEqueryEtryPt_doHUP ENDqueryEtryPt @@ -892,7 +843,14 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); + INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); + DBGPRINTF("omfile: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not "); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID)); @@ -901,8 +859,9 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"filecreatemode", 0, eCmdHdlrFileCreateMode, NULL, &fCreateMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"createdirs", 0, eCmdHdlrBinary, NULL, &bCreateDirs, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"failonchownfailure", 0, eCmdHdlrBinary, NULL, &bFailOnChown, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileForceChown", 0, eCmdHdlrBinary, NULL, &bForceChown, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionfileenablesync", 0, eCmdHdlrBinary, NULL, &bEnableSync, STD_LOADABLE_MODULE_ID)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionfiledefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszTplName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionfiledefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszFileDfltTplName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit /* vi:set ai: diff --git a/tools/omfile.h b/tools/omfile.h index 03e081f3..8dca6a88 100644 --- a/tools/omfile.h +++ b/tools/omfile.h @@ -3,7 +3,7 @@ * * File begun on 2007-07-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -28,7 +28,11 @@ /* prototypes */ rsRetVal modInitFile(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); +/* the define below is dirty, but we need it for ompipe integration. There is no + * other way to have the functionality (well, one way would be to go through the + * globals, but that seems not yet justified. -- rgerhards, 2010-03-01 + */ +uchar *pszFileDfltTplName; #endif /* #ifndef OMFILE_H_INCLUDED */ -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/tools/omfwd.c b/tools/omfwd.c index 88a382e0..1e9e6c27 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -10,7 +10,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -48,6 +48,7 @@ #endif #include <pthread.h> #include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "net.h" @@ -82,12 +83,15 @@ typedef struct _instanceData { permittedPeers_t *pPermPeers; int iStrmDrvrMode; char *f_hname; - int *pSockArray; /* sockets to use for UDP */ + int *pSockArray; /* sockets to use for UDP */ int bIsConnected; /* are we connected to remote host? 0 - no, 1 - yes, UDP means addr resolved */ struct addrinfo *f_addr; - int compressionLevel; /* 0 - no compression, else level for zlib */ + int compressionLevel; /* 0 - no compression, else level for zlib */ char *port; int protocol; + int iUDPRebindInterval; /* rebind interval */ + int iTCPRebindInterval; /* rebind interval */ + int nXmit; /* number of transmissions since last (re-)bind */ # define FORW_UDP 0 # define FORW_TCP 1 /* following fields for TCP-based delivery */ @@ -100,9 +104,32 @@ static uchar *pszStrmDrvr = NULL; /* name of the stream driver to use */ static short iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ static short bResendLastOnRecon = 0; /* should the last message be re-sent on a successful reconnect? */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ +static int iUDPRebindInterval = 0; /* support for automatic re-binding (load balancers!). 0 - no rebind */ +static int iTCPRebindInterval = 0; /* support for automatic re-binding (load balancers!). 0 - no rebind */ static permittedPeers_t *pPermPeers = NULL; +static rsRetVal doTryResume(instanceData *pData); + +/* Close the UDP sockets. + * rgerhards, 2009-05-29 + */ +static rsRetVal +closeUDPSockets(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + if(pData->pSockArray != NULL) { + net.closeUDPListenSockets(pData->pSockArray); + pData->pSockArray = NULL; + freeaddrinfo(pData->f_addr); + pData->f_addr = NULL; + } +pData->bIsConnected = 0; // TODO: remove this variable altogether + RETiRet; +} + + /* get the syslog forward port from selector_t. The passed in * struct must be one that is setup for forwarding. * rgerhards, 2007-06-28 @@ -148,30 +175,19 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - if(pData->f_addr != NULL) { /* TODO: is the check ok? */ - freeaddrinfo(pData->f_addr); - pData->f_addr = NULL; - } - if(pData->port != NULL) - free(pData->port); - /* final cleanup */ DestructTCPInstanceData(pData); - if(pData->pSockArray != NULL) - net.closeUDPListenSockets(pData->pSockArray); + closeUDPSockets(pData); if(pData->protocol == FORW_TCP) { tcpclt.Destruct(&pData->pTCPClt); } - if(pData->f_hname != NULL) - free(pData->f_hname); - if(pData->pszStrmDrvr != NULL) - free(pData->pszStrmDrvr); - if(pData->pszStrmDrvrAuthMode != NULL) - free(pData->pszStrmDrvrAuthMode); - if(pData->pPermPeers != NULL) - net.DestructPermittedPeers(&pData->pPermPeers); + free(pData->port); + free(pData->f_hname); + free(pData->pszStrmDrvr); + free(pData->pszStrmDrvrAuthMode); + net.DestructPermittedPeers(&pData->pPermPeers); ENDfreeInstance @@ -192,6 +208,16 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) unsigned lsent = 0; int bSendSuccess; + if(pData->iUDPRebindInterval && (pData->nXmit++ % pData->iUDPRebindInterval == 0)) { + dbgprintf("omfwd dropping UDP 'connection' (as configured)\n"); + pData->nXmit = 1; /* else we have an addtl wrap at 2^31-1 */ + CHKiRet(closeUDPSockets(pData)); + } + + if(pData->pSockArray == NULL) { + CHKiRet(doTryResume(pData)); + } + if(pData->pSockArray != NULL) { /* we need to track if we have success sending to the remote * peer. Success is indicated by at least one sendto() call @@ -224,6 +250,7 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) } } +finalize_it: RETiRet; } @@ -409,13 +436,13 @@ CODESTARTdoAction * hard-coded but this may be changed to a config parameter. * rgerhards, 2006-11-30 */ - if(pData->compressionLevel && (l > MIN_SIZE_FOR_COMPRESS)) { + if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { Bytef *out; - uLongf destLen = sizeof(out) / sizeof(Bytef); + uLongf destLen = iMaxLine + iMaxLine/100 +12; /* recommended value from zlib doc */ uLong srcLen = l; int ret; /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ - CHKmalloc(out = (Bytef*) malloc(iMaxLine + iMaxLine/100 + 12)); + CHKmalloc(out = (Bytef*) MALLOC(destLen)); out[0] = 'z'; out[1] = '\0'; ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, @@ -431,11 +458,14 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ dbgprintf("Compression failed, sending uncompressed message\n"); + free(out); } else if(destLen+1 < l) { /* only use compression if there is a gain in using it! */ dbgprintf("there is gain in compression, so we do it\n"); psz = (char*) out; l = destLen + 1; /* take care for the "z" at message start! */ + } else { + free(out); } ++destLen; } @@ -456,6 +486,12 @@ CODESTARTdoAction } } finalize_it: +# ifdef USE_NETZIP + if(psz != (char*) ppString[0]) { + /* we need to free temporary buffer, alloced above - Naoya Nakazawa, 2010-01-11 */ + free(psz); + } +# endif ENDdoAction @@ -590,7 +626,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) tmp = ++p; for(i=0 ; *p && isdigit((int) *p) ; ++p, ++i) /* SKIP AND COUNT */; - pData->port = malloc(i + 1); + pData->port = MALLOC(i + 1); if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store syslog forwarding port, " "using default port, results may not be what you intend\n"); @@ -616,6 +652,10 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) CHKmalloc(pData->f_hname = strdup((char*) q)); } + /* copy over config data as needed */ + pData->iUDPRebindInterval = iUDPRebindInterval; + pData->iTCPRebindInterval = iTCPRebindInterval; + /* process template */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_TraditionalForwardFormat" : pszTplName)); @@ -629,6 +669,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) CHKiRet(tcpclt.SetSendFrame(pData->pTCPClt, TCPSendFrame)); CHKiRet(tcpclt.SetSendPrepRetry(pData->pTCPClt, TCPSendPrepRetry)); CHKiRet(tcpclt.SetFraming(pData->pTCPClt, tcp_framing)); + CHKiRet(tcpclt.SetRebindInterval(pData->pTCPClt, pData->iTCPRebindInterval)); pData->iStrmDrvrMode = iStrmDrvrMode; if(pszStrmDrvr != NULL) CHKmalloc(pData->pszStrmDrvr = (uchar*)strdup((char*)pszStrmDrvr)); @@ -699,6 +740,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a /* we now must reset all non-string values */ iStrmDrvrMode = 0; bResendLastOnRecon = 0; + iUDPRebindInterval = 0; + iTCPRebindInterval = 0; return RS_RET_OK; } @@ -713,6 +756,8 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(net,LM_NET_FILENAME)); CHKiRet(regCfSysLineHdlr((uchar *)"actionforwarddefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszTplName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionsendtcprebindinterval", 0, eCmdHdlrInt, NULL, &iTCPRebindInterval, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionsendudprebindinterval", 0, eCmdHdlrInt, NULL, &iUDPRebindInterval, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszStrmDrvr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdrivermode", 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdriverauthmode", 0, eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, NULL)); diff --git a/tools/ompipe.c b/tools/ompipe.c new file mode 100644 index 00000000..7cf61822 --- /dev/null +++ b/tools/ompipe.c @@ -0,0 +1,240 @@ +/* ompipe.c + * This is the implementation of the build-in pipe output module. + * Note that this module stems back to the "old" (4.4.2 and below) + * omfile. There were some issues with the new omfile code and pipes + * (namely in regard to xconsole), so we took out the pipe code and moved + * that to a separate module. That a) immediately solves the issue for a + * less common use case and probably makes it much easier to enhance + * file and pipe support (now independently) in the future (we always + * needed to think about pipes in omfile so far, what we now no longer + * need to, hopefully resulting in reduction of complexity). + * + * NOTE: read comments in module-template.h to understand how this pipe + * works! + * + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include <fcntl.h> +#include <sys/file.h> + +#include "syslogd.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "ompipe.h" +#include "omfile.h" /* for dirty trick: access to $ActionFileDefaultTemplate value */ +#include "cfsysline.h" +#include "module-template.h" +#include "conf.h" +#include "errmsg.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + + +/* globals for default values */ +/* end globals for default values */ + + +typedef struct _instanceData { + uchar f_fname[MAXFNAME];/* pipe or template name (display only) */ + short fd; /* pipe descriptor for (current) pipe */ +} instanceData; + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + dbgprintf("pipe %s", pData->f_fname); + if (pData->fd == -1) + dbgprintf(" (unused)"); +ENDdbgPrintInstInfo + + +/* This is now shared code for all types of files. It simply prepares + * pipe access, which, among others, means the the pipe wil be opened + * and any directories in between will be created (based on config, of + * course). -- rgerhards, 2008-10-22 + * changed to iRet interface - 2009-03-19 + */ +static inline rsRetVal +preparePipe(instanceData *pData) +{ + DEFiRet; + pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); + RETiRet; +} + + +/* rgerhards 2004-11-11: write to a pipe output. This + * will be called for all outputs using pipe semantics, + * for example also for pipes. + */ +static rsRetVal writePipe(uchar **ppString, instanceData *pData) +{ + int iLenWritten; + DEFiRet; + + ASSERT(pData != NULL); + + if(pData->fd == -1) { + rsRetVal iRetLocal; + iRetLocal = preparePipe(pData); + if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + } + + /* create the message based on format specified */ + iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); + if(iLenWritten < 0) { + int e = errno; + char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + DBGPRINTF("pipe (%d) write error %d: %s\n", pData->fd, e, errStr); + + /* If a named pipe is full, we suspend this action for a while */ + if(e == EAGAIN) + ABORT_FINALIZE(RS_RET_SUSPENDED); + + close(pData->fd); + pData->fd = -1; /* tell that fd is no longer open! */ + iRet = RS_RET_SUSPENDED; + errno = e; + errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); + } + +finalize_it: + RETiRet; +} + + +BEGINcreateInstance +CODESTARTcreateInstance + pData->fd = -1; +ENDcreateInstance + + +BEGINfreeInstance +CODESTARTfreeInstance + if(pData->fd != -1) + close(pData->fd); +ENDfreeInstance + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + DBGPRINTF(" (%s)\n", pData->f_fname); + iRet = writePipe(ppString, pData); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct + /* yes, the if below is redundant, but I need it now. Will go away as + * the code further changes. -- rgerhards, 2007-07-25 + */ + if(*p == '|') { + if((iRet = createInstance(&pData)) != RS_RET_OK) { + ENDfunc + return iRet; /* this can not use RET_iRet! */ + } + } else { + /* this is not clean, but we need it for the time being + * TODO: remove when cleaning up modularization + */ + ENDfunc + return RS_RET_CONFLINE_UNPROCESSED; + } + + CODE_STD_STRING_REQUESTparseSelectorAct(1) + ++p; + /* rgerhards 2004-11-17: from now, we need to have different + * processing, because after the first comma, the template name + * to use is specified. So we need to scan for the first coma first + * and then look at the rest of the line. + */ + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); + + /* at this stage, we ignore the return value of preparePipe, this is taken + * care of in later steps. -- rgerhards, 2009-03-19 + */ + preparePipe(pData); + + if(pData->fd < 0 ) { + pData->fd = -1; + DBGPRINTF("Error opening log pipe: %s\n", pData->f_fname); + errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output pipe '%s'", pData->f_fname); + } +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINdoHUP +CODESTARTdoHUP + if(pData->fd != -1) { + close(pData->fd); + pData->fd = -1; + } +ENDdoHUP + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_doHUP +ENDqueryEtryPt + + +BEGINmodInit(Pipe) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +ENDmodInit +/* vi:set ai: + */ diff --git a/tools/ompipe.h b/tools/ompipe.h new file mode 100644 index 00000000..d17346c5 --- /dev/null +++ b/tools/ompipe.h @@ -0,0 +1,31 @@ +/* ompipe.h + * These are the definitions for the build-in pipe output module. + * + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. + * + * This pipe is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the pipe "COPYING" in this distribution. + */ +#ifndef OMPIPE_H_INCLUDED +#define OMPIPE_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitPipe(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef OMPIPE_H_INCLUDED */ +/* vi:set ai: + */ diff --git a/tools/omshell.c b/tools/omshell.c index 7b815869..f8a68527 100644 --- a/tools/omshell.c +++ b/tools/omshell.c @@ -38,7 +38,7 @@ #include <stdlib.h> #include <string.h> #include <assert.h> -#include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "omshell.h" diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c index 830bbc87..a89297d7 100644 --- a/tools/omusrmsg.c +++ b/tools/omusrmsg.c @@ -50,7 +50,11 @@ #include <assert.h> #include <signal.h> #include <sys/param.h> -#include <utmp.h> +#ifdef HAVE_UTMP_H +# include <utmp.h> +#else +# include <utmpx.h> +#endif #include <unistd.h> #include <sys/uio.h> #include <sys/stat.h> @@ -66,7 +70,7 @@ #include "srUtils.h" #include "stringbuf.h" #include "syslogd-types.h" -#include "syslogd.h" +#include "conf.h" #include "omusrmsg.h" #include "module-template.h" #include "errmsg.h" diff --git a/tools/pmrfc3164.c b/tools/pmrfc3164.c new file mode 100644 index 00000000..38f556a2 --- /dev/null +++ b/tools/pmrfc3164.c @@ -0,0 +1,240 @@ +/* pmrfc3164.c + * This is a parser module for RFC3164(legacy syslog)-formatted messages. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2009-11-04 by RGerhards + * + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include <ctype.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.rfc3164") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +/* parse a legay-formatted syslog message. + */ +BEGINparse + uchar *p2parse; + int lenMsg; + int bTAGCharDetected; + int i; /* general index for parsing */ + uchar bufParseTAG[CONF_TAG_MAXSIZE]; + uchar bufParseHOSTNAME[CONF_HOSTNAME_MAXSIZE]; +CODESTARTparse + dbgprintf("Message will now be parsed by the legacy syslog parser (one size fits all... ;)).\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + setProtocolVersion(pMsg, 0); + + /* Check to see if msg contains a timestamp. We start by assuming + * that the message timestamp is the time of reception (which we + * generated ourselfs and then try to actually find one inside the + * message. There we go from high-to low precison and are done + * when we find a matching one. -- rgerhards, 2008-09-16 + */ + if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* we are done - parse pointer is moved by ParseTIMESTAMP3339 */; + } else if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; + } else if(*p2parse == ' ' && lenMsg > 1) { /* try to see if it is slighly malformed - HP procurve seems to do that sometimes */ + ++p2parse; /* move over space */ + --lenMsg; + if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* indeed, we got it! */ + /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; + } else {/* parse pointer needs to be restored, as we moved it off-by-one + * for this try. + */ + --p2parse; + ++lenMsg; + } + } + + if(pMsg->msgFlags & IGNDATE) { + /* we need to ignore the msg data, so simply copy over reception date */ + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + } + + /* rgerhards, 2006-03-13: next, we parse the hostname and tag. But we + * do this only when the user has not forbidden this. I now introduce some + * code that allows a user to configure rsyslogd to treat the rest of the + * message as MSG part completely. In this case, the hostname will be the + * machine that we received the message from and the tag will be empty. This + * is meant to be an interim solution, but for now it is in the code. + */ + if(bParseHOSTNAMEandTAG && !(pMsg->msgFlags & INTERNAL_MSG)) { + /* parse HOSTNAME - but only if this is network-received! + * rger, 2005-11-14: we still have a problem with BSD messages. These messages + * do NOT include a host name. In most cases, this leads to the TAG to be treated + * as hostname and the first word of the message as the TAG. Clearly, this is not + * of advantage ;) I think I have now found a way to handle this situation: there + * are certain characters which are frequently used in TAG (e.g. ':'), which are + * *invalid* in host names. So while parsing the hostname, I check for these characters. + * If I find them, I set a simple flag but continue. After parsing, I check the flag. + * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change + * the fields. I think this logic shall work with any type of syslog message. + * rgerhards, 2009-06-23: and I now have extended this logic to every character + * that is not a valid hostname. + */ + bTAGCharDetected = 0; + if(lenMsg > 0 && pMsg->msgFlags & PARSE_HOSTNAME) { + i = 0; + while(i < lenMsg && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.' + || p2parse[i] == '_' || p2parse[i] == '-') && i < (CONF_HOSTNAME_MAXSIZE - 1)) { + bufParseHOSTNAME[i] = p2parse[i]; + ++i; + } + + if(i == lenMsg) { + /* we have a message that is empty immediately after the hostname, + * but the hostname thus is valid! -- rgerhards, 2010-02-22 + */ + p2parse += i; + lenMsg -= i; + bufParseHOSTNAME[i] = '\0'; + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); + } else if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) { + /* we got a hostname! */ + p2parse += i + 1; /* "eat" it (including SP delimiter) */ + lenMsg -= i + 1; + bufParseHOSTNAME[i] = '\0'; + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); + } + } + + /* now parse TAG - that should be present in message from all sources. + * This code is somewhat not compliant with RFC 3164. As of 3164, + * the TAG field is ended by any non-alphanumeric character. In + * practice, however, the TAG often contains dashes and other things, + * which would end the TAG. So it is not desirable. As such, we only + * accept colon and SP to be terminators. Even there is a slight difference: + * a colon is PART of the TAG, while a SP is NOT part of the tag + * (it is CONTENT). Starting 2008-04-04, we have removed the 32 character + * size limit (from RFC3164) on the tag. This had bad effects on existing + * envrionments, as sysklogd didn't obey it either (probably another bug + * in RFC3164...). We now receive the full size, but will modify the + * outputs so that only 32 characters max are used by default. + */ + i = 0; + while(lenMsg > 0 && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *p2parse++; + --lenMsg; + } + if(lenMsg > 0 && *p2parse == ':') { + ++p2parse; + --lenMsg; + bufParseTAG[i++] = ':'; + } + + /* no TAG can only be detected if the message immediatly ends, in which case an empty TAG + * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23 + */ + bufParseTAG[i] = '\0'; /* terminate string */ + MsgSetTAG(pMsg, bufParseTAG, i); + } else {/* we enter this code area when the user has instructed rsyslog NOT + * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 + */ + if(!(pMsg->msgFlags & INTERNAL_MSG)) { + DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); + } + } + + /* The rest is the actual MSG */ + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(pmrfc3164) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("rfc3164 parser init called\n"); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ diff --git a/tools/pmrfc3164.h b/tools/pmrfc3164.h new file mode 100644 index 00000000..24304094 --- /dev/null +++ b/tools/pmrfc3164.h @@ -0,0 +1,33 @@ +/* pmrfc3164.h + * These are the definitions for the RFC3164 parser module. + * + * File begun on 2009-11-04 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef PMRFC3164_H_INCLUDED +#define PMRFC3164_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitpmrfc3164(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef PMRFC3164_H_INCLUDED */ +/* vi:set ai: + */ diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c new file mode 100644 index 00000000..07994ade --- /dev/null +++ b/tools/pmrfc5424.c @@ -0,0 +1,329 @@ +/* pmrfc5424.c + * This is a parser module for RFC5424-formatted messages. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2009-11-03 by RGerhards + * + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.rfc5424") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* config data */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +/* Helper to parseRFCSyslogMsg. This function parses a field up to + * (and including) the SP character after it. The field contents is + * returned in a caller-provided buffer. The parsepointer is advanced + * to after the terminating SP. The caller must ensure that the + * provided buffer is large enough to hold the to be extracted value. + * Returns 0 if everything is fine or 1 if either the field is not + * SP-terminated or any other error occurs. -- rger, 2005-11-24 + * The function now receives the size of the string and makes sure + * that it does not process more than that. The *pLenStr counter is + * updated on exit. -- rgerhards, 2009-09-23 + */ +static int parseRFCField(uchar **pp2parse, uchar *pResult, int *pLenStr) +{ + uchar *p2parse; + int iRet = 0; + + assert(pp2parse != NULL); + assert(*pp2parse != NULL); + assert(pResult != NULL); + + p2parse = *pp2parse; + + /* this is the actual parsing loop */ + while(*pLenStr > 0 && *p2parse != ' ') { + *pResult++ = *p2parse++; + --(*pLenStr); + } + + if(*pLenStr > 0 && *p2parse == ' ') { + ++p2parse; /* eat SP, but only if not at end of string */ + --(*pLenStr); + } else { + iRet = 1; /* there MUST be an SP! */ + } + *pResult = '\0'; + + /* set the new parse pointer */ + *pp2parse = p2parse; + return 0; +} + + +/* Helper to parseRFCSyslogMsg. This function parses the structured + * data field of a message. It does NOT parse inside structured data, + * just gets the field as whole. Parsing the single entities is left + * to other functions. The parsepointer is advanced + * to after the terminating SP. The caller must ensure that the + * provided buffer is large enough to hold the to be extracted value. + * Returns 0 if everything is fine or 1 if either the field is not + * SP-terminated or any other error occurs. -- rger, 2005-11-24 + * The function now receives the size of the string and makes sure + * that it does not process more than that. The *pLenStr counter is + * updated on exit. -- rgerhards, 2009-09-23 + */ +static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr) +{ + uchar *p2parse; + int bCont = 1; + int iRet = 0; + int lenStr; + + assert(pp2parse != NULL); + assert(*pp2parse != NULL); + assert(pResult != NULL); + + p2parse = *pp2parse; + lenStr = *pLenStr; + + /* this is the actual parsing loop + * Remeber: structured data starts with [ and includes any characters + * until the first ] followed by a SP. There may be spaces inside + * structured data. There may also be \] inside the structured data, which + * do NOT terminate an element. + */ + if(lenStr == 0 || *p2parse != '[') + return 1; /* this is NOT structured data! */ + + if(*p2parse == '-') { /* empty structured data? */ + *pResult++ = '-'; + ++p2parse; + --lenStr; + } else { + while(bCont) { + if(lenStr < 2) { + /* we now need to check if we have only structured data */ + if(lenStr > 0 && *p2parse == ']') { + *pResult++ = *p2parse; + p2parse++; + lenStr--; + bCont = 0; + } else { + iRet = 1; /* this is not valid! */ + bCont = 0; + } + } else if(*p2parse == '\\' && *(p2parse+1) == ']') { + /* this is escaped, need to copy both */ + *pResult++ = *p2parse++; + *pResult++ = *p2parse++; + lenStr -= 2; + } else if(*p2parse == ']' && *(p2parse+1) == ' ') { + /* found end, just need to copy the ] and eat the SP */ + *pResult++ = *p2parse; + p2parse += 2; + lenStr -= 2; + bCont = 0; + } else { + *pResult++ = *p2parse++; + --lenStr; + } + } + } + + if(lenStr > 0 && *p2parse == ' ') { + ++p2parse; /* eat SP, but only if not at end of string */ + --lenStr; + } else { + iRet = 1; /* there MUST be an SP! */ + } + *pResult = '\0'; + + /* set the new parse pointer */ + *pp2parse = p2parse; + *pLenStr = lenStr; + return 0; +} + +/* parse a RFC5424-formatted syslog message. This function returns + * 0 if processing of the message shall continue and 1 if something + * went wrong and this messe should be ignored. This function has been + * implemented in the effort to support syslog-protocol. Please note that + * the name (parse *RFC*) stems from the hope that syslog-protocol will + * some time become an RFC. Do not confuse this with informational + * RFC 3164 (which is legacy syslog). + * + * currently supported format: + * + * <PRI>VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP [SD-ID]s SP MSG + * + * <PRI> is already stripped when this function is entered. VERSION already + * has been confirmed to be "1", but has NOT been stripped from the message. + * + * rger, 2005-11-24 + */ +//static int parseRFCSyslogMsg(msg_t *pMsg, int flags) +BEGINparse + uchar *p2parse; + uchar *pBuf = NULL; + int lenMsg; + int bContParse = 1; +CODESTARTparse + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; + + /* check if we are the right parser */ + if(lenMsg < 2 || p2parse[0] != '1' || p2parse[1] != ' ') { + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + DBGPRINTF("Message has RFC5424/syslog-protocol format.\n"); + setProtocolVersion(pMsg, 1); + p2parse += 2; + lenMsg -= 2; + + /* Now get us some memory we can use as a work buffer while parsing. + * We simply allocated a buffer sufficiently large to hold all of the + * message, so we can not run into any troubles. I think this is + * wiser than to use individual buffers. + */ + CHKmalloc(pBuf = MALLOC(sizeof(uchar) * (lenMsg + 1))); + + /* IMPORTANT NOTE: + * Validation is not actually done below nor are any errors handled. I have + * NOT included this for the current proof of concept. However, it is strongly + * advisable to add it when this code actually goes into production. + * rgerhards, 2005-11-24 + */ + + /* TIMESTAMP */ + if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + if(pMsg->msgFlags & IGNDATE) { + /* we need to ignore the msg data, so simply copy over reception date */ + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + } + } else { + DBGPRINTF("no TIMESTAMP detected!\n"); + bContParse = 0; + } + + /* HOSTNAME */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetHOSTNAME(pMsg, pBuf, ustrlen(pBuf)); + } + + /* APP-NAME */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetAPPNAME(pMsg, (char*)pBuf); + } + + /* PROCID */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetPROCID(pMsg, (char*)pBuf); + } + + /* MSGID */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetMSGID(pMsg, (char*)pBuf); + } + + /* STRUCTURED-DATA */ + if(bContParse) { + parseRFCStructuredData(&p2parse, pBuf, &lenMsg); + MsgSetStructuredData(pMsg, (char*)pBuf); + } + + /* MSG */ + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); + +finalize_it: + if(pBuf != NULL) + free(pBuf); +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(pmrfc5424) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("rfc5424 parser init called\n"); + dbgprintf("GetParserName addr %p\n", GetParserName); +ENDmodInit + +/* vim:set ai: + */ diff --git a/tools/pmrfc5424.h b/tools/pmrfc5424.h new file mode 100644 index 00000000..df2a1c81 --- /dev/null +++ b/tools/pmrfc5424.h @@ -0,0 +1,33 @@ +/* pmrfc5424.h + * These are the definitions for the RFCC5424 parser module. + * + * File begun on 2009-11-03 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef PMRFC54254_H_INCLUDED +#define PMRFC54254_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitpmrfc5424(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef PMRFC54254_H_INCLUDED */ +/* vi:set ai: + */ diff --git a/tools/rsyslog.conf.5 b/tools/rsyslog.conf.5 index 0a2422c6..e8a4ab92 100644 --- a/tools/rsyslog.conf.5 +++ b/tools/rsyslog.conf.5 @@ -80,7 +80,7 @@ used like this: .IP $ModLoad imudp .IP -$InputUDPServerRun 514 +$UDPServerRun 514 .TP .I imtcp Input plugin for plain TCP syslog. Replaces the deprecated -t @@ -164,8 +164,8 @@ a pattern of facilities and priorities belonging to the specified action. The selector field itself again consists of two parts, a facility and a priority, separated by a period ('.'). Both parts are case insensitive and can also be specified as decimal numbers, but don't do that, you have been warned. -Both facilities and priorities are described in rsyslog(3). The names mentioned -below correspond to the similar LOG_-values in /usr/include/rsyslog.h. +Both facilities and priorities are described in syslog(3). The names mentioned +below correspond to the similar LOG_-values in /usr/include/syslog.h. The facility is one of the following keywords: auth, authpriv, cron, daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, user, uucp and diff --git a/tools/rsyslogd.8 b/tools/rsyslogd.8 index 7abcf97a..6ac30e46 100644 --- a/tools/rsyslogd.8 +++ b/tools/rsyslogd.8 @@ -234,34 +234,31 @@ reacts to a set of signals. You may easily send a signal to using the following: .IP .nf -kill -SIGNAL $(cat /var/run/syslogd.pid) +kill -SIGNAL $(cat /var/run/rsyslogd.pid) .fi .PP Note that -SIGNAL must be replaced with the actual signal you are trying to send, e.g. with HUP. So it then becomes: .IP .nf -kill -HUP $(cat /var/run/syslogd.pid) +kill -HUP $(cat /var/run/rsyslogd.pid) .fi .PP .TP .B HUP This lets .B rsyslogd -perform a re-initialization. All open files are closed, the -configuration file (default is -.IR /etc/rsyslog.conf ")" -will be reread and the -.BR rsyslog (3) -facility is started again. +perform close all open files. +Also, in v3 a full restart will be done in order to read changed configuration files. Note that this means a full rsyslogd restart is done. This has, among others, the consequence that TCP and other connections are torn down. Also, if any queues are not running in disk assisted mode or are not set to persist data on shutdown, queue data is lost. HUPing rsyslogd is an extremely expensive operation and should only be done when actually necessary. Actually, it is -a rsyslgod stop immediately followed by a restart. Future versions will probably -include a special handling which only closes files, but will not cause any -of the other effects. +a rsyslgod stop immediately followed by a restart. Future versions will remove +this restart functionality of HUP (it will go away in v5). So it is advised to use +HUP only for closing files, and a "real restart" (e.g. /etc/rc.d/rsyslogd restart) +to activate configuration changes. .TP .B TERM ", " INT ", " QUIT .B Rsyslogd diff --git a/tools/syslogd.c b/tools/syslogd.c index 0b860448..1ba3ef2b 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -16,12 +16,9 @@ * parts of the code have been rewritten. * * This Project was intiated and is maintained by - * Rainer Gerhards <rgerhards@hq.adiscon.com>. See - * AUTHORS to learn who helped make it become a reality. + * Rainer Gerhards <rgerhards@hq.adiscon.com>. * - * If you have questions about rsyslogd in general, please email - * info@adiscon.com. To learn more about rsyslogd, please visit - * http://www.rsyslog.com. + * For further information, please see http://www.rsyslog.com * * \author Rainer Gerhards <rgerhards@adiscon.com> * \date 2003-10-17 @@ -36,7 +33,7 @@ * to the database). * * rsyslog - An Enhanced syslogd Replacement. - * Copyright 2003-2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2003-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -127,66 +124,55 @@ #include "omusrmsg.h" #include "omfwd.h" #include "omfile.h" +#include "ompipe.h" #include "omdiscard.h" +#include "pmrfc5424.h" +#include "pmrfc3164.h" #include "threads.h" +#include "wti.h" #include "queue.h" #include "stream.h" #include "conf.h" -#include "vm.h" #include "errmsg.h" #include "datetime.h" #include "parser.h" -#include "sysvar.h" +#include "batch.h" #include "unicode-helper.h" +#include "ruleset.h" +#include "rule.h" +#include "net.h" +#include "vm.h" +#include "prop.h" /* definitions for objects we access */ DEFobjCurrIf(obj) DEFobjCurrIf(glbl) -DEFobjCurrIf(datetime) +DEFobjCurrIf(datetime) /* TODO: make go away! */ DEFobjCurrIf(conf) DEFobjCurrIf(expr) -DEFobjCurrIf(vm) -DEFobjCurrIf(var) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +DEFobjCurrIf(rule) +DEFobjCurrIf(ruleset) +DEFobjCurrIf(prop) +DEFobjCurrIf(parser) DEFobjCurrIf(net) /* TODO: make go away! */ /* forward definitions */ static rsRetVal GlobalClassExit(void); -/* We define our own set of syslog defintions so that we - * do not need to rely on (possibly different) implementations. - * 2007-07-19 rgerhards - */ -/* missing definitions for solaris - * 2006-02-16 Rger - */ -#ifdef __sun -# define LOG_AUTHPRIV LOG_AUTH -#endif -#define INTERNAL_NOPRI 0x10 /* the "no priority" priority */ -#define LOG_FTP (11<<3) /* ftp daemon */ - - -#ifndef UTMP_FILE -#ifdef UTMP_FILENAME -#define UTMP_FILE UTMP_FILENAME -#else -#ifdef _PATH_UTMP -#define UTMP_FILE _PATH_UTMP -#else -#define UTMP_FILE "/etc/utmp" -#endif -#endif -#endif #ifndef _PATH_LOGCONF #define _PATH_LOGCONF "/etc/rsyslog.conf" #endif #ifndef _PATH_MODDIR -#define _PATH_MODDIR "/lib/rsyslog/" +# if defined(__FreeBSD__) +# define _PATH_MODDIR "/usr/local/lib/rsyslog/" +# else +# define _PATH_MODDIR "/lib/rsyslog/" +# endif #endif #if defined(SYSLOGD_PIDNAME) @@ -215,14 +201,12 @@ static rsRetVal GlobalClassExit(void); # endif #endif -#ifndef _PATH_DEV -# define _PATH_DEV "/dev/" -#endif - #ifndef _PATH_TTY -#define _PATH_TTY "/dev/tty" +# define _PATH_TTY "/dev/tty" #endif +static prop_t *pInternalInputName = NULL; /* there is only one global inputName for all internally-generated messages */ +static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static uchar *ConfFile = (uchar*) _PATH_LOGCONF; /* read-only after startup */ static char *PidFile = _PATH_LOGPID; /* read-only after startup */ @@ -230,8 +214,6 @@ static pid_t myPid; /* our pid for use in self-generated messages, e.g. on start /* mypid is read-only after the initial fork() */ static int bHadHUP = 0; /* did we have a HUP? */ -static int bParseHOSTNAMEandTAG = 1; /* global config var: should the hostname and tag be - * parsed inside message - rgerhards, 2006-03-13 */ static int bFinished = 0; /* used by termination signal handler, read-only except there * is either 0 or the number of the signal that requested the * termination. @@ -247,8 +229,6 @@ int repeatinterval[2] = { 30, 60 }; /* # of secs before flush */ #define LIST_DELIMITER ':' /* delimiter between two hosts */ -struct filed *Files = NULL; /* read-only after init() (but beware of sigusr1!) */ - static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */ typedef struct legacyOptsLL_s { @@ -258,19 +238,18 @@ typedef struct legacyOptsLL_s { legacyOptsLL_t *pLegacyOptsLL = NULL; /* global variables for config file state */ -int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ int iCompatibilityMode = 0; /* version we should be compatible with; 0 means sysklogd. It is the default, so if no -c<n> option is given, we make ourselvs as compatible to sysklogd as possible. */ +#define DFLT_bLogStatusMsgs 1 +static int bLogStatusMsgs = DFLT_bLogStatusMsgs; /* log rsyslog start/stop/HUP messages? */ static int bDebugPrintTemplateList = 1;/* output template list in debug mode? */ static int bDebugPrintCfSysLineHandlerList = 1;/* output cfsyslinehandler list in debug mode? */ static int bDebugPrintModuleList = 1;/* output module list in debug mode? */ -uchar cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */ -int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ static int bErrMsgToStderr = 1; /* print error messages to stderr (in addition to everything else)? */ int bReduceRepeatMsgs; /* reduce repeated message - 0 - no, 1 - yes */ +int bAbortOnUncleanConfig = 0; /* abort run (rather than starting with partial config) if there was any issue in conf */ int bActExecWhenPrevSusp; /* execute action only when previous one was suspended? */ -int iActExecOnceInterval = 0; /* execute action once every nn seconds */ /* end global config file state variables */ int MarkInterval = 20 * 60; /* interval between marks in seconds - read-only after startup */ @@ -287,7 +266,7 @@ extern int errno; static uchar *pszConfDAGFile = NULL; /* name of config DAG file, non-NULL means generate one */ /* main message queue and its configuration parameters */ -static qqueue_t *pMsgQueue = NULL; /* the main message queue */ +qqueue_t *pMsgQueue = NULL; /* the main message queue */ static int iMainMsgQueueSize = 10000; /* size of the main message queue above */ static int iMainMsgQHighWtrMark = 8000; /* high water mark for disk-assisted queues */ static int iMainMsgQLowWtrMark = 2000; /* low water mark for disk-assisted queues */ @@ -298,58 +277,32 @@ static queueType_t MainMsgQueType = QUEUETYPE_FIXED_ARRAY; /* type of the main m static uchar *pszMainMsgQFName = NULL; /* prefix for the main message queue file */ static int64 iMainMsgQueMaxFileSize = 1024*1024; static int iMainMsgQPersistUpdCnt = 0; /* persist queue info every n updates */ -static int iMainMsgQtoQShutdown = 0; /* queue shutdown */ +static int bMainMsgQSyncQeueFiles = 0; /* sync queue files on every write? */ +static int iMainMsgQtoQShutdown = 1500; /* queue shutdown (ms) */ static int iMainMsgQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iMainMsgQtoEnq = 2000; /* timeout for queue enque */ static int iMainMsgQtoWrkShutdown = 60000; /* timeout for worker thread shutdown */ static int iMainMsgQWrkMinMsgs = 100; /* minimum messages per worker needed to start a new one */ static int iMainMsgQDeqSlowdown = 0; /* dequeue slowdown (simple rate limiting) */ static int64 iMainMsgQueMaxDiskSpace = 0; /* max disk space allocated 0 ==> unlimited */ +static int64 iMainMsgQueDeqBatchSize = 32; /* dequeue batch size */ static int bMainMsgQSaveOnShutdown = 1; /* save queue on shutdown (when DA enabled)? */ static int iMainMsgQueueDeqtWinFromHr = 0; /* hour begin of time frame when queue is to be dequeued */ static int iMainMsgQueueDeqtWinToHr = 25; /* hour begin of time frame when queue is to be dequeued */ -/* support for simple textual representation of FIOP names - * rgerhards, 2005-09-27 - */ -static char* getFIOPName(unsigned iFIOP) -{ - char *pRet; - switch(iFIOP) { - case FIOP_CONTAINS: - pRet = "contains"; - break; - case FIOP_ISEQUAL: - pRet = "isequal"; - break; - case FIOP_STARTSWITH: - pRet = "startswith"; - break; - case FIOP_REGEX: - pRet = "regex"; - break; - default: - pRet = "NOP"; - break; - } - return pRet; -} - - /* Reset config variables to default values. * rgerhards, 2007-07-17 */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { - cCCEscapeChar = '#'; + bLogStatusMsgs = DFLT_bLogStatusMsgs; bActExecWhenPrevSusp = 0; - iActExecOnceInterval = 0; bDebugPrintTemplateList = 1; bDebugPrintCfSysLineHandlerList = 1; bDebugPrintModuleList = 1; - bEscapeCCOnRcv = 1; /* default is to escape control characters */ bReduceRepeatMsgs = 0; + bAbortOnUncleanConfig = 0; free(pszMainMsgQFName); pszMainMsgQFName = NULL; iMainMsgQueueSize = 10000; @@ -360,7 +313,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iMainMsgQueMaxFileSize = 1024 * 1024; iMainMsgQueueNumWorkers = 1; iMainMsgQPersistUpdCnt = 0; - iMainMsgQtoQShutdown = 0; + bMainMsgQSyncQeueFiles = 0; + iMainMsgQtoQShutdown = 1500; iMainMsgQtoActShutdown = 1000; iMainMsgQtoEnq = 2000; iMainMsgQtoWrkShutdown = 60000; @@ -369,6 +323,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bMainMsgQSaveOnShutdown = 1; MainMsgQueType = QUEUETYPE_FIXED_ARRAY; iMainMsgQueMaxDiskSpace = 0; + iMainMsgQueDeqBatchSize = 32; glbliActionResumeRetryCount = 0; return RS_RET_OK; @@ -395,8 +350,6 @@ static char **crunch_list(char *list); static void reapchild(); static void debug_switch(); static void sighup_handler(); -static void freeSelectors(void); -static void processImInternal(void); static int usage(void) @@ -404,7 +357,7 @@ static int usage(void) fprintf(stderr, "usage: rsyslogd [-c<version>] [-46AdnqQvwx] [-l<hostlist>] [-s<domainlist>]\n" " [-f<conffile>] [-i<pidfile>] [-N<level>] [-M<module load path>]\n" " [-u<number>]\n" - "To run rsyslogd in native mode, use \"rsyslogd -c3 <other options>\"\n\n" + "To run rsyslogd in native mode, use \"rsyslogd -c5 <other options>\"\n\n" "For further information see http://www.rsyslog.com/doc\n"); exit(1); /* "good" exit - done to terminate usage() */ } @@ -416,13 +369,16 @@ static int usage(void) */ /* return back the approximate current number of messages in the main message queue + * This number includes the messages that reside in an associated DA queue (if + * it exists) -- rgerhards, 2009-10-14 */ rsRetVal diagGetMainMsgQSize(int *piSize) { DEFiRet; assert(piSize != NULL); - *piSize = pMsgQueue->iQueueSize; + *piSize = (pMsgQueue->pqDA != NULL) ? pMsgQueue->pqDA->iQueueSize : 0; + *piSize += pMsgQueue->iQueueSize; RETiRet; } @@ -430,69 +386,8 @@ diagGetMainMsgQSize(int *piSize) /* ------------------------------ end support functions for imdiag ------------------------------ */ -/* function to destruct a selector_t object - * rgerhards, 2007-08-01 - */ -rsRetVal -selectorDestruct(void *pVal) -{ - selector_t *pThis = (selector_t *) pVal; - - assert(pThis != NULL); - - if(pThis->pCSHostnameComp != NULL) - rsCStrDestruct(&pThis->pCSHostnameComp); - if(pThis->pCSProgNameComp != NULL) - rsCStrDestruct(&pThis->pCSProgNameComp); - - if(pThis->f_filter_type == FILTER_PROP) { - if(pThis->f_filterData.prop.pCSPropName != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); - if(pThis->f_filterData.prop.pCSCompValue != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); - if(pThis->f_filterData.prop.regex_cache != NULL) - rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); - } else if(pThis->f_filter_type == FILTER_EXPR) { - if(pThis->f_filterData.f_expr != NULL) - expr.Destruct(&pThis->f_filterData.f_expr); - } - - llDestroy(&pThis->llActList); - free(pThis); - - return RS_RET_OK; -} - - -/* function to construct a selector_t object - * rgerhards, 2007-08-01 - */ -rsRetVal -selectorConstruct(selector_t **ppThis) -{ - DEFiRet; - selector_t *pThis; - - assert(ppThis != NULL); - - if((pThis = (selector_t*) calloc(1, sizeof(selector_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); - -finalize_it: - if(iRet != RS_RET_OK) { - if(pThis != NULL) { - selectorDestruct(pThis); - } - } - *ppThis = pThis; - RETiRet; -} - - /* rgerhards, 2005-10-24: crunch_list is called only during option processing. So - * it is never called once rsyslogd is running (not even when HUPed). This code + * it is never called once rsyslogd is running. This code * contains some exits, but they are considered safe because they only happen * during startup. Anyhow, when we review the code here, we might want to * reconsider the exit()s. @@ -520,7 +415,7 @@ static char **crunch_list(char *list) for (count=i=0; p[i]; i++) if (p[i] == LIST_DELIMITER) count++; - if ((result = (char **)malloc(sizeof(char *) * (count+2))) == NULL) { + if ((result = (char **)MALLOC(sizeof(char *) * (count+2))) == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ } @@ -532,7 +427,7 @@ static char **crunch_list(char *list) */ count = 0; while ((q=strchr(p, LIST_DELIMITER))) { - result[count] = (char *) malloc((q - p + 1) * sizeof(char)); + result[count] = (char *) MALLOC((q - p + 1) * sizeof(char)); if (result[count] == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ @@ -543,7 +438,7 @@ static char **crunch_list(char *list) count++; } if ((result[count] = \ - (char *)malloc(sizeof(char) * strlen(p) + 1)) == NULL) { + (char *)MALLOC(sizeof(char) * strlen(p) + 1)) == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ } @@ -553,7 +448,7 @@ static char **crunch_list(char *list) #if 0 count=0; while (result[count]) - dbgprintf("#%d: %s\n", count, StripDomains[count++]); + DBGPRINTF("#%d: %s\n", count, StripDomains[count++]); #endif return result; } @@ -562,7 +457,7 @@ static char **crunch_list(char *list) void untty(void) #ifdef HAVE_SETSID { - if ( !Debug ) { + if(!Debug) { setsid(); } return; @@ -571,352 +466,61 @@ void untty(void) { int i; - if ( !Debug ) { + if(!Debug) { i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) - (void) ioctl(i, (int) TIOCNOTTY, (char *)0); + (void) ioctl(i, (int) TIOCNOTTY, NULL); # else /* TODO: we need to implement something for HP UX! -- rgerhards, 2008-03-04 */ /* actually, HP UX should have setsid, so the code directly above should * trigger. So the actual question is why it doesn't do that... */ # endif - (void) close(i); + close(i); } } } #endif -/* Take a raw input line, decode the message, and print the message - * on the appropriate log files. - * rgerhards 2004-11-08: Please note - * that this function does only a partial decoding. At best, it splits - * the PRI part. No further decode happens. The rest is done in - * logmsg(). - * Added the iSource parameter so that we know if we have to parse - * HOSTNAME or not. rgerhards 2004-11-16. - * changed parameter iSource to bParseHost. For details, see comment in - * printchopped(). rgerhards 2005-10-06 - * rgerhards: 2008-03-06: added "flags" to allow an input module to specify - * flags, most importantly to request ignoring the messages' timestamp. - * - * rgerhards, 2008-03-19: - * I added an additional calling parameter to permit specifying the flow - * control capability of the source. - * - * rgerhards, 2008-05-16: - * I added an additional calling parameter (hnameIP) to enable specifying the IP - * of a remote host. - * - * rgerhards, 2008-09-11: - * Interface change: added new parameter "InputName", permits the input to provide - * a string that identifies it. May be NULL, but must be a valid char* pointer if - * non-NULL. - * - * rgerhards, 2008-10-06: - * Interface change: added new parameter "stTime", which enables the caller to provide - * a timestamp that is to be used as timegenerated instead of the current system time. - * This is meant to facilitate performance optimization. Some inputs support such modes. - * If stTime is NULL, the current system time is used. - * - * rgerhards, 2008-10-09: - * interface change: bParseHostname removed, now in flags +/* This takes a received message that must be decoded and submits it to + * the main message queue. This is a legacy function which is being provided + * to aid older input plugins that do not support message creation via + * the new interfaces themselves. It is not recommended to use this + * function for new plugins. -- rgerhards, 2009-10-12 */ -static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) +rsRetVal +parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType, + prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime) { - DEFiRet; - register uchar *p; - int pri; + prop_t *pProp = NULL; msg_t *pMsg; + DEFiRet; - /* Now it is time to create the message object (rgerhards) */ + /* we now create our own message object and submit it to the queue */ if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } - if(pszInputName != NULL) - MsgSetInputName(pMsg, pszInputName); + if(pInputName != NULL) + MsgSetInputName(pMsg, pInputName); + MsgSetRawMsg(pMsg, (char*)msg, len); MsgSetFlowControlType(pMsg, flowCtlType); - MsgSetRawMsg(pMsg, (char*)msg); - - /* test for special codes */ - pri = DEFUPRI; - p = msg; - if (*p == '<') { - pri = 0; - while (isdigit((int) *++p)) - { - pri = 10 * pri + (*p - '0'); - } - if (*p == '>') - ++p; - } - if (pri &~ (LOG_FACMASK|LOG_PRIMASK)) - pri = DEFUPRI; - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); + pMsg->msgFlags = flags | NEEDS_PARSING; - /* Now we look at the HOSTNAME. That is a bit complicated... - * If we have a locally received message, it does NOT - * contain any hostname information in the message itself. - * As such, the HOSTNAME is the same as the system that - * the message was received from (that, for obvious reasons, - * being the local host). rgerhards 2004-11-16 - */ - if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) - MsgSetHOSTNAME(pMsg, hname); - MsgSetRcvFrom(pMsg, hname); - CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); - - /* rgerhards 2004-11-19: well, well... we've now seen that we - * have the "hostname problem" also with the traditional Unix - * message. As we like to emulate it, we need to add the hostname - * to it. - */ - if(MsgSetUxTradMsg(pMsg, (char*)p) != 0) - ABORT_FINALIZE(RS_RET_ERR); - - logmsg(pMsg, flags); + MsgSetRcvFromStr(pMsg, hname, ustrlen(hname), &pProp); + CHKiRet(prop.Destruct(&pProp)); + CHKiRet(MsgSetRcvFromIPStr(pMsg, hnameIP, ustrlen(hnameIP), &pProp)); + CHKiRet(prop.Destruct(&pProp)); + CHKiRet(submitMsg(pMsg)); finalize_it: RETiRet; } -/* This takes a received message that must be decoded and submits it to - * the main message queue. The function calls the necessary parser. - * - * rgerhards, 2006-11-30: I have greatly changed this function. Formerly, - * it tried to reassemble multi-part messages, which is a legacy stock - * sysklogd concept. In essence, that was that messages not ending with - * \0 were glued together. As far as I can see, this is a sysklogd - * specific feature and, from looking at the code, seems to be used - * pretty seldom (if at all). I remove this now, not the least because it is totally - * incompatible with upcoming IETF syslog standards. If you experience - * strange behaviour with messages beeing split across multiple lines, - * this function here might be the place to look at. - * - * Some previous history worth noting: - * I added the "iSource" parameter. This is needed to distinguish between - * messages that have a hostname in them (received from the internet) and - * those that do not have (most prominently /dev/log). rgerhards 2004-11-16 - * And now I removed the "iSource" parameter and changed it to be "bParseHost", - * because all that it actually controls is whether the host is parsed or not. - * For rfc3195 support, we needed to modify the algo for host parsing, so we can - * no longer rely just on the source (rfc3195d forwarded messages arrive via - * unix domain sockets but contain the hostname). rgerhards, 2005-10-06 - * - * rgerhards, 2008-02-18: - * This function was previously called "printchopped"() and has been renamed - * as part of the effort to create a clean internal message submission interface. - * It also has been adopted to our usual calling interface, but currently does - * not provide any useful return states. But we now have the hook and things can - * improve in the future. <-- TODO! - * - * rgerhards, 2008-03-19: - * I added an additional calling parameter to permit specifying the flow - * control capability of the source. - * - * rgerhards, 2008-05-16: - * I added an additional calling parameter (hnameIP) to enable specifying the IP - * of a remote host. - * - * rgerhards, 2008-09-11: - * Interface change: added new parameter "InputName", permits the input to provide - * a string that identifies it. May be NULL, but must be a valid char* pointer if - * non-NULL. - * - * rgerhards, 2008-10-06: - * Interface change: added new parameter "stTime", which enables the caller to provide - * a timestamp that is to be used as timegenerated instead of the current system time. - * This is meant to facilitate performance optimization. Some inputs support such modes. - * If stTime is NULL, the current system time is used. - * - * rgerhards, 2008-10-09: - * interface change: bParseHostname removed, now in flags - */ -rsRetVal -parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) -{ - DEFiRet; - register int iMsg; - uchar *pMsg; - uchar *pData; - uchar *pEnd; - int iMaxLine; - uchar *tmpline = NULL; -# ifdef USE_NETZIP - uchar *deflateBuf = NULL; - uLongf iLenDefBuf; -# endif - - assert(hname != NULL); - assert(hnameIP != NULL); - assert(msg != NULL); - assert(len >= 0); - - /* we first allocate work buffers large enough to hold the configured maximum - * size of a message. Over time, we should change this to a more optimal way, i.e. - * by calling the function with the actual length of the message to be parsed. - * rgerhards, 2008-09-02 - * - * TODO: optimize buffer handling */ - iMaxLine = glbl.GetMaxLine(); - CHKmalloc(tmpline = malloc(sizeof(uchar) * (iMaxLine + 1))); - - /* we first check if we have a NUL character at the very end of the - * message. This seems to be a frequent problem with a number of senders. - * So I have now decided to drop these NULs. However, if they are intentional, - * that may cause us some problems, e.g. with syslog-sign. On the other hand, - * current code always has problems with intentional NULs (as it needs to escape - * them to prevent problems with the C string libraries), so that does not - * really matter. Just to be on the save side, we'll log destruction of such - * NULs in the debug log. - * rgerhards, 2007-09-14 - */ - if(*(msg + len - 1) == '\0') { - dbgprintf("dropped NUL at very end of message\n"); - len--; - } - - /* then we check if we need to drop trailing LFs, which often make - * their way into syslog messages unintentionally. In order to remain - * compatible to recent IETF developments, we allow the user to - * turn on/off this handling. rgerhards, 2007-07-23 - */ - if(bDropTrailingLF && *(msg + len - 1) == '\n') { - dbgprintf("dropped LF at very end of message (DropTrailingLF is set)\n"); - len--; - } - - iMsg = 0; /* initialize receiving buffer index */ - pMsg = tmpline; /* set receiving buffer pointer */ - pData = msg; /* set source buffer pointer */ - pEnd = msg + len; /* this is one off, which is intensional */ - -# ifdef USE_NETZIP - /* we first need to check if we have a compressed record. If so, - * we must decompress it. - */ - if(len > 0 && *msg == 'z') { /* compressed data present? (do NOT change order if conditions!) */ - /* we have compressed data, so let's deflate it. We support a maximum - * message size of iMaxLine. If it is larger, an error message is logged - * and the message is dropped. We do NOT try to decompress larger messages - * as such might be used for denial of service. It might happen to later - * builds that such functionality be added as an optional, operator-configurable - * feature. - */ - int ret; - iLenDefBuf = iMaxLine; - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); - ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); - dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", - ret, (long) iLenDefBuf, len-1); - /* Now check if the uncompression worked. If not, there is not much we can do. In - * that case, we log an error message but ignore the message itself. Storing the - * compressed text is dangerous, as it contains control characters. So we do - * not do this. If someone would like to have a copy, this code here could be - * modified to do a hex-dump of the buffer in question. We do not include - * this functionality right now. - * rgerhards, 2006-12-07 - */ - if(ret != Z_OK) { - errmsg.LogError(0, NO_ERRCODE, "Uncompression of a message failed with return code %d " - "- enable debug logging if you need further information. " - "Message ignored.", ret); - FINALIZE; /* unconditional exit, nothing left to do... */ - } - pData = deflateBuf; - pEnd = deflateBuf + iLenDefBuf; - } -# else /* ifdef USE_NETZIP */ - /* in this case, we still need to check if the message is compressed. If so, we must - * tell the user we can not accept it. - */ - if(len > 0 && *msg == 'z') { - errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " - "support enabled. The message will be ignored."); - FINALIZE; - } -# endif /* ifdef USE_NETZIP */ - - while(pData < pEnd) { - if(iMsg >= iMaxLine) { - /* emergency, we now need to flush, no matter if - * we are at end of message or not... - */ - if(iMsg == iMaxLine) { - *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); - } else { - /* This case in theory never can happen. If it happens, we have - * a logic error. I am checking for it, because if I would not, - * we would address memory invalidly with the code above. I - * do not care much about this case, just a debug log entry - * (I couldn't do any more smart things anyway...). - * rgerhards, 2007-9-20 - */ - dbgprintf("internal error: iMsg > max msg size in printchopped()\n"); - } - FINALIZE; /* in this case, we are done... nothing left we can do */ - } - if(*pData == '\0') { /* guard against \0 characters... */ - /* changed to the sequence (somewhat) proposed in - * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 - */ - if(iMsg + 3 < iMaxLine) { /* do we have space? */ - *(pMsg + iMsg++) = cCCEscapeChar; - *(pMsg + iMsg++) = '0'; - *(pMsg + iMsg++) = '0'; - *(pMsg + iMsg++) = '0'; - } /* if we do not have space, we simply ignore the '\0'... */ - /* log an error? Very questionable... rgerhards, 2006-11-30 */ - /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ - ++pData; - } else if(bEscapeCCOnRcv && iscntrl((int) *pData)) { - /* we are configured to escape control characters. Please note - * that this most probably break non-western character sets like - * Japanese, Korean or Chinese. rgerhards, 2007-07-17 - * Note: sysklogd logs octal values only for DEL and CCs above 127. - * For others, it logs ^n where n is the control char converted to an - * alphabet character. We like consistency and thus escape it to octal - * in all cases. If someone complains, we may change the mode. At least - * we known now what's going on. - * rgerhards, 2007-07-17 - */ - if(iMsg + 3 < iMaxLine) { /* do we have space? */ - *(pMsg + iMsg++) = cCCEscapeChar; - *(pMsg + iMsg++) = '0' + ((*pData & 0300) >> 6); - *(pMsg + iMsg++) = '0' + ((*pData & 0070) >> 3); - *(pMsg + iMsg++) = '0' + ((*pData & 0007)); - } /* again, if we do not have space, we ignore the char - see comment at '\0' */ - ++pData; - } else { - *(pMsg + iMsg++) = *pData++; - } - } - - *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - - /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); - -finalize_it: - if(tmpline != NULL) - free(tmpline); -# ifdef USE_NETZIP - if(deflateBuf != NULL) - free(deflateBuf); -# endif - RETiRet; -} - - /* this is a special function used to submit an error message. This * function is also passed to the runtime library as the generic error * message handler. -- rgerhards, 2008-04-17 @@ -931,13 +535,7 @@ submitErrMsg(int iErr, uchar *msg) /* rgerhards 2004-11-09: the following is a function that can be used - * to log a message orginating from the syslogd itself. In sysklogd code, - * this is done by simply calling logmsg(). However, logmsg() is changed in - * rsyslog so that it takes a msg "object". So it can no longer be called - * directly. This method here solves the need. It provides an interface that - * allows to construct a locally-generated message. Please note that this - * function here probably is only an interim solution and that we need to - * think on the best way to do this. + * to log a message orginating from the syslogd itself. */ rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags) @@ -947,26 +545,26 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd")); - MsgSetUxTradMsg(pMsg, (char*)msg); - MsgSetRawMsg(pMsg, (char*)msg); - MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); - MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); - MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); + MsgSetInputName(pMsg, pInternalInputName); + MsgSetRawMsgWOSize(pMsg, (char*)msg); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); + MsgSetRcvFromIP(pMsg, pLocalHostIP); + MsgSetMSGoffs(pMsg, 0); /* check if we have an error code associated and, if so, - * adjust the tag. -- r5gerhards, 2008-06-27 + * adjust the tag. -- rgerhards, 2008-06-27 */ if(iErr == NO_ERRCODE) { - MsgSetTAG(pMsg, "rsyslogd:"); + MsgSetTAG(pMsg, UCHAR_CONSTANT("rsyslogd:"), sizeof("rsyslogd:") - 1); } else { - snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr); + size_t len = snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr); pszTag[32] = '\0'; /* just to make sure... */ - MsgSetTAG(pMsg, (char*)pszTag); + MsgSetTAG(pMsg, pszTag, len); } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - pMsg->bParseHOSTNAME = 0; flags |= INTERNAL_MSG; + pMsg->msgFlags = flags; /* we now check if we should print internal messages out to stderr. This was * suggested by HKS as a way to help people troubleshoot rsyslog configuration @@ -976,746 +574,185 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) * permits us to process unmodified config files which otherwise contain a * supressor statement. */ - if(bErrMsgToStderr || iConfigVerify) { - fprintf(stderr, "rsyslogd: %s\n", msg); + if(((Debug == DEBUG_FULL || NoFork) && bErrMsgToStderr) || iConfigVerify) { + if(LOG_PRI(pri) == LOG_ERR) + fprintf(stderr, "rsyslogd: %s\n", msg); } if(bHaveMainQueue == 0) { /* not yet in queued mode */ - iminternalAddMsg(pri, pMsg, flags); + iminternalAddMsg(pri, pMsg); } else { /* we have the queue, so we can simply provide the * message to the queue engine. */ - logmsg(pMsg, flags); + submitMsg(pMsg); } finalize_it: RETiRet; } -/* This functions looks at the given message and checks if it matches the - * provided filter condition. If so, it returns true, else it returns - * false. This is a helper to logmsg() and meant to drive the decision - * process if a message is to be processed or not. As I expect this - * decision code to grow more complex over time AND logmsg() is already - * a very lengthy function, I thought a separate function is more appropriate. - * 2005-09-19 rgerhards - * 2008-02-25 rgerhards: changed interface, now utilizes iRet, bProcessMsg - * returns is message should be procesed. +/* check message against ACL set + * rgerhards, 2009-11-16 */ -static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProcessMsg) -{ - DEFiRet; - unsigned short pbMustBeFreed; - char *pszPropVal; - int bRet = 0; - vm_t *pVM = NULL; - var_t *pResult = NULL; - - assert(f != NULL); - assert(pMsg != NULL); - - /* we first have a look at the global, BSD-style block filters (for tag - * and host). Only if they match, we evaluate the actual filter. - * rgerhards, 2005-10-18 - */ - if(f->eHostnameCmpMode == HN_NO_COMP) { - /* EMPTY BY INTENSION - we check this value first, because - * it is the one most often used, so this saves us time! - */ - } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { - if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - dbgprintf("hostname filter '+%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } else { /* must be -hostname */ - if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - dbgprintf("hostname filter '-%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } - - if(f->pCSProgNameComp != NULL) { - int bInv = 0, bEqv = 0, offset = 0; - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp)) == '-') { - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp) + 1) == '-') - offset = 1; - else { - bInv = 1; - offset = 1; +#if 0 +static inline rsRetVal +chkMsgAgainstACL() { + /* if we reach this point, we had a good receive and can process the packet received */ + /* check if we have a different sender than before, if so, we need to query some new values */ + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us + * syslog messages. If it isn't, we do not further + * process the message but log a warning (if we are + * configured to do this). + * rgerhards, 2005-09-26 + */ + *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", + (struct sockaddr *)&frominet, (char*)fromHostFQDN); + + if(!*pbIsPermitted) { + DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender %s discarded", + (char*)fromHost); + } } } - if(!rsCStrOffsetSzStrCmp(f->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) - bEqv = 1; - - if((!bEqv && !bInv) || (bEqv && bInv)) { - /* not equal or inverted selection, so we are already done... */ - dbgprintf("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); - FINALIZE; - } - } - - /* done with the BSD-style block filters */ - - if(f->f_filter_type == FILTER_PRI) { - /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ - ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - } else if(f->f_filter_type == FILTER_EXPR) { - CHKiRet(vm.Construct(&pVM)); - CHKiRet(vm.ConstructFinalize(pVM)); - CHKiRet(vm.SetMsg(pVM, pMsg)); - CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); - CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); - /* VM is destructed on function exit */ - bRet = (pResult->val.num) ? 1 : 0; - } else { - assert(f->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, f->f_filterData.prop.pCSPropName, &pbMustBeFreed); - - /* Now do the compares (short list currently ;)) */ - switch(f->f_filterData.prop.operation ) { - case FIOP_CONTAINS: - if(rsCStrLocateInSzStr(f->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) - bRet = 1; - break; - case FIOP_ISEQUAL: - if(rsCStrSzStrCmp(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_STARTSWITH: - if(rsCStrSzStrStartsWithCStr(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - case FIOP_EREREGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - default: - /* here, it handles NOP (for performance reasons) */ - assert(f->f_filterData.prop.operation == FIOP_NOP); - bRet = 1; /* as good as any other default ;) */ - break; - } - - /* now check if the value must be negated */ - if(f->f_filterData.prop.isNegated) - bRet = (bRet == 1) ? 0 : 1; - - if(Debug) { - dbgprintf("Filter: check for property '%s' (value '%s') ", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName), - pszPropVal); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("%s '%s': %s\n", - getFIOPName(f->f_filterData.prop.operation), - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue), - bRet ? "TRUE" : "FALSE"); - } - - /* cleanup */ - if(pbMustBeFreed) - free(pszPropVal); } - -finalize_it: - /* destruct in any case, not just on error, but it makes error handling much easier */ - if(pVM != NULL) - vm.Destruct(&pVM); - - if(pResult != NULL) - var.Destruct(&pResult); - - *bProcessMsg = bRet; - RETiRet; } +#endif -/* helper to processMsg(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 +/* consumes a single messages - this function is primarily used to shuffle + * out some code from msgConsumer(). After this function, the message is + * (by definition!) considered committed. + * rgerhards, 2009-11-16 */ -typedef struct processMsgDoActions_s { - int bPrevWasSuspended; /* was the previous action suspended? */ - msg_t *pMsg; -} processMsgDoActions_t; -DEFFUNC_llExecFunc(processMsgDoActions) -{ +static inline rsRetVal +msgConsumeOne(msg_t *pMsg, prop_t **propFromHost, prop_t **propFromHostIP) { + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + int bIsPermitted; DEFiRet; - rsRetVal iRetMod; /* return value of module - we do not always pass that back */ - action_t *pAction = (action_t*) pData; - processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; - assert(pAction != NULL); - - if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - dbgprintf("not calling action because the previous one is not suspended\n"); - ABORT_FINALIZE(RS_RET_OK); + if((pMsg->msgFlags & NEEDS_ACLCHK_U) != 0) { + dbgprintf("msgConsumer: UDP ACL must be checked for message (hostname-based)\n"); + CHKiRet(net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP)); + bIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)pMsg->rcvFrom.pfrominet, (char*)fromHostFQDN, 1); + if(!bIsPermitted) { + DBGPRINTF("Message from '%s' discarded, not a permitted sender host\n", + fromHostFQDN); + ABORT_FINALIZE(RS_RET_ERR); + /* save some of the info we obtained */ + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), propFromHostIP)); + pMsg->msgFlags &= ~NEEDS_ACLCHK_U; + } } - iRetMod = actionCallAction(pAction, pDoActData->pMsg); - if(iRetMod == RS_RET_DISCARDMSG) { - ABORT_FINALIZE(RS_RET_DISCARDMSG); - } else if(iRetMod == RS_RET_SUSPENDED) { - /* indicate suspension for next module to be called */ - pDoActData->bPrevWasSuspended = 1; - } else { - pDoActData->bPrevWasSuspended = 0; - } + if((pMsg->msgFlags & NEEDS_PARSING) != 0) + CHKiRet(parser.ParseMsg(pMsg)); + ruleset.ProcessMsg(pMsg); finalize_it: RETiRet; } -/* Process (consume) a received message. Calls the actions configured. - * rgerhards, 2005-10-13 - */ -static void -processMsg(msg_t *pMsg) -{ - selector_t *f; - int bContinue; - int bProcessMsg; - processMsgDoActions_t DoActData; - rsRetVal iRet; - - BEGINfunc - assert(pMsg != NULL); - - /* log the message to the particular outputs */ - - bContinue = 1; - for (f = Files; f != NULL && bContinue ; f = f->f_next) { - /* first check the filters... */ - iRet = shouldProcessThisMessage(f, pMsg, &bProcessMsg); - if(!bProcessMsg) { - continue; - } - - /* ok -- from here, we have action-specific code, nothing really selector-specific -- rger 2007-08-01 */ - DoActData.pMsg = pMsg; - DoActData.bPrevWasSuspended = 0; - if(llExecFunc(&f->llActList, processMsgDoActions, (void*)&DoActData) == RS_RET_DISCARDMSG) - bContinue = 0; - } - ENDfunc -} - - /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE - * THREAD. - * Please note: the message object is destructed by the queue itself! + * THREAD. It receives an array of pointers, which it must iterate + * over. We do not do any further batching, as this is of no benefit + * for the main queue. */ static rsRetVal -msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) +msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShutdownImmediate) { + int i; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; DEFiRet; - msg_t *pMsg = (msg_t*) pUsr; - assert(pMsg != NULL); + assert(pBatch != NULL); - if((pMsg->msgFlags & NEEDS_PARSING) != 0) { - parseMsg(pMsg); + for(i = 0 ; i < pBatch->nElem && !*pbShutdownImmediate ; i++) { + DBGPRINTF("msgConsumer processes msg %d/%d\n", i, pBatch->nElem); + msgConsumeOne((msg_t*) pBatch->pElem[i].pUsrp, &propFromHost, &propFromHostIP); + pBatch->pElem[i].state = BATCH_STATE_COMM; } - processMsg(pMsg); - msgDestruct(&pMsg); + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); RETiRet; } -/* Helper to parseRFCSyslogMsg. This function parses a field up to - * (and including) the SP character after it. The field contents is - * returned in a caller-provided buffer. The parsepointer is advanced - * to after the terminating SP. The caller must ensure that the - * provided buffer is large enough to hold the to be extracted value. - * Returns 0 if everything is fine or 1 if either the field is not - * SP-terminated or any other error occurs. - * rger, 2005-11-24 - */ -static int parseRFCField(uchar **pp2parse, uchar *pResult) -{ - uchar *p2parse; - int iRet = 0; - - assert(pp2parse != NULL); - assert(*pp2parse != NULL); - assert(pResult != NULL); - - p2parse = *pp2parse; - - /* this is the actual parsing loop */ - while(*p2parse && *p2parse != ' ') { - *pResult++ = *p2parse++; - } - - if(*p2parse == ' ') - ++p2parse; /* eat SP, but only if not at end of string */ - else - iRet = 1; /* there MUST be an SP! */ - *pResult = '\0'; - - /* set the new parse pointer */ - *pp2parse = p2parse; - return 0; -} - - -/* Helper to parseRFCSyslogMsg. This function parses the structured - * data field of a message. It does NOT parse inside structured data, - * just gets the field as whole. Parsing the single entities is left - * to other functions. The parsepointer is advanced - * to after the terminating SP. The caller must ensure that the - * provided buffer is large enough to hold the to be extracted value. - * Returns 0 if everything is fine or 1 if either the field is not - * SP-terminated or any other error occurs. - * rger, 2005-11-24 - */ -static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult) -{ - uchar *p2parse; - int bCont = 1; - int iRet = 0; - - assert(pp2parse != NULL); - assert(*pp2parse != NULL); - assert(pResult != NULL); - - p2parse = *pp2parse; - - /* this is the actual parsing loop - * Remeber: structured data starts with [ and includes any characters - * until the first ] followed by a SP. There may be spaces inside - * structured data. There may also be \] inside the structured data, which - * do NOT terminate an element. - */ - if(*p2parse != '[') - return 1; /* this is NOT structured data! */ - - while(bCont) { - if(*p2parse == '\0') { - iRet = 1; /* this is not valid! */ - bCont = 0; - } else if(*p2parse == '\\' && *(p2parse+1) == ']') { - /* this is escaped, need to copy both */ - *pResult++ = *p2parse++; - *pResult++ = *p2parse++; - } else if(*p2parse == ']' && *(p2parse+1) == ' ') { - /* found end, just need to copy the ] and eat the SP */ - *pResult++ = *p2parse; - p2parse += 2; - bCont = 0; - } else { - *pResult++ = *p2parse++; - } - } - - if(*p2parse == ' ') - ++p2parse; /* eat SP, but only if not at end of string */ - else - iRet = 1; /* there MUST be an SP! */ - *pResult = '\0'; - - /* set the new parse pointer */ - *pp2parse = p2parse; - return 0; -} - -/* parse a RFC-formatted syslog message. This function returns - * 0 if processing of the message shall continue and 1 if something - * went wrong and this messe should be ignored. This function has been - * implemented in the effort to support syslog-protocol. Please note that - * the name (parse *RFC*) stems from the hope that syslog-protocol will - * some time become an RFC. Do not confuse this with informational - * RFC 3164 (which is legacy syslog). - * - * currently supported format: - * - * <PRI>VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP [SD-ID]s SP MSG - * - * <PRI> is already stripped when this function is entered. VERSION already - * has been confirmed to be "1", but has NOT been stripped from the message. - * - * rger, 2005-11-24 - */ -int parseRFCSyslogMsg(msg_t *pMsg, int flags) -{ - uchar *p2parse; - uchar *pBuf; - int bContParse = 1; - - BEGINfunc - assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; - - /* do a sanity check on the version and eat it */ - assert(p2parse[0] == '1' && p2parse[1] == ' '); - p2parse += 2; - - /* Now get us some memory we can use as a work buffer while parsing. - * We simply allocated a buffer sufficiently large to hold all of the - * message, so we can not run into any troubles. I think this is - * more wise then to use individual buffers. - */ - if((pBuf = malloc(sizeof(uchar) * ustrlen(p2parse) + 1)) == NULL) - return 1; - - /* IMPORTANT NOTE: - * Validation is not actually done below nor are any errors handled. I have - * NOT included this for the current proof of concept. However, it is strongly - * advisable to add it when this code actually goes into production. - * rgerhards, 2005-11-24 - */ - - /* TIMESTAMP */ - if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse) == RS_RET_OK) { - if(flags & IGNDATE) { - /* we need to ignore the msg data, so simply copy over reception date */ - memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); - } - } else { - dbgprintf("no TIMESTAMP detected!\n"); - bContParse = 0; - } - - /* HOSTNAME */ - if(bContParse) { - parseRFCField(&p2parse, pBuf); - MsgSetHOSTNAME(pMsg, pBuf); - } else { - /* we can not parse, so we get the system we - * received the data from. - */ - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); - } - - /* APP-NAME */ - if(bContParse) { - parseRFCField(&p2parse, pBuf); - MsgSetAPPNAME(pMsg, (char*)pBuf); - } - - /* PROCID */ - if(bContParse) { - parseRFCField(&p2parse, pBuf); - MsgSetPROCID(pMsg, (char*)pBuf); - } - - /* MSGID */ - if(bContParse) { - parseRFCField(&p2parse, pBuf); - MsgSetMSGID(pMsg, (char*)pBuf); - } - - /* STRUCTURED-DATA */ - if(bContParse) { - parseRFCStructuredData(&p2parse, pBuf); - MsgSetStructuredData(pMsg, (char*)pBuf); - } - - /* MSG */ - MsgSetMSG(pMsg, (char*)p2parse); - - free(pBuf); - ENDfunc - return 0; /* all ok */ -} - - -/* parse a legay-formatted syslog message. This function returns - * 0 if processing of the message shall continue and 1 if something - * went wrong and this messe should be ignored. This function has been - * implemented in the effort to support syslog-protocol. - * rger, 2005-11-24 - * As of 2006-01-10, I am removing the logic to continue parsing only - * when a valid TIMESTAMP is detected. Validity of other fields already - * is ignored. This is due to the fact that the parser has grown smarter - * and is now more able to understand different dialects of the syslog - * message format. I do not expect any bad side effects of this change, - * but I thought I log it in this comment. - * rgerhards, 2006-01-10 - */ -int parseLegacySyslogMsg(msg_t *pMsg, int flags) -{ - uchar *p2parse; - char *pBuf; - char *pWork; - cstr_t *pStrB; - int iCnt; - int bTAGCharDetected; - BEGINfunc - - assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; - - /* Check to see if msg contains a timestamp. We start by assuming - * that the message timestamp is the time of reciption (which we - * generated ourselfs and then try to actually find one inside the - * message. There we go from high-to low precison and are done - * when we find a matching one. -- rgerhards, 2008-09-16 - */ - if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse) == RS_RET_OK) { - /* we are done - parse pointer is moved by ParseTIMESTAMP3339 */; - } else if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse) == RS_RET_OK) { - /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; - } else if(*p2parse == ' ') { /* try to see if it is slighly malformed - HP procurve seems to do that sometimes */ - ++p2parse; /* move over space */ - if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse) == RS_RET_OK) { - /* indeed, we got it! */ - /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; - } else { - /* parse pointer needs to be restored, as we moved it off-by-one - * for this try. - */ - --p2parse; - } - } - - if(flags & IGNDATE) { - /* we need to ignore the msg data, so simply copy over reception date */ - memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); - } - - /* rgerhards, 2006-03-13: next, we parse the hostname and tag. But we - * do this only when the user has not forbidden this. I now introduce some - * code that allows a user to configure rsyslogd to treat the rest of the - * message as MSG part completely. In this case, the hostname will be the - * machine that we received the message from and the tag will be empty. This - * is meant to be an interim solution, but for now it is in the code. - */ - if(bParseHOSTNAMEandTAG && !(flags & INTERNAL_MSG)) { - /* parse HOSTNAME - but only if this is network-received! - * rger, 2005-11-14: we still have a problem with BSD messages. These messages - * do NOT include a host name. In most cases, this leads to the TAG to be treated - * as hostname and the first word of the message as the TAG. Clearly, this is not - * of advantage ;) I think I have now found a way to handle this situation: there - * are certain characters which are frequently used in TAG (e.g. ':'), which are - * *invalid* in host names. So while parsing the hostname, I check for these characters. - * If I find them, I set a simple flag but continue. After parsing, I check the flag. - * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change - * the fields. I think this logic shall work with any type of syslog message. - */ - bTAGCharDetected = 0; - if(flags & PARSE_HOSTNAME) { - /* TODO: quick and dirty memory allocation */ - /* the memory allocated is far too much in most cases. But on the plus side, - * it is quite fast... - rgerhards, 2007-09-20 - */ - if((pBuf = malloc(sizeof(char)* (ustrlen(p2parse) +1))) == NULL) - return 1; - pWork = pBuf; - /* this is the actual parsing loop */ - while(*p2parse && *p2parse != ' ' && *p2parse != ':') { - if(*p2parse == '[' || *p2parse == ']' || *p2parse == '/') - bTAGCharDetected = 1; - *pWork++ = *p2parse++; - } - /* we need to handle ':' seperately, because it terminates the - * TAG - so we also need to terminate the parser here! - * rgerhards, 2007-09-10 *p2parse points to a valid address here in - * any case. We can reach this point only if we are at end of string, - * or we have a ':' or ' '. What the if below does is check if we are - * not at end of string and, if so, advance the parse pointer. If we - * are already at end of string, *p2parse is equal to '\0', neither if - * will be true and the parse pointer remain as is. This is perfectly - * well. - */ - if(*p2parse == ':') { - bTAGCharDetected = 1; - /* We will move hostname to tag, so preserve ':' (otherwise we - * will needlessly change the message format) */ - *pWork++ = *p2parse++; - } else if(*p2parse == ' ') - ++p2parse; - *pWork = '\0'; - MsgAssignHOSTNAME(pMsg, pBuf); - } - /* check if we seem to have a TAG */ - if(bTAGCharDetected) { - /* indeed, this smells like a TAG, so lets use it for this. We take - * the HOSTNAME from the sender system instead. - */ - dbgprintf("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); - moveHOSTNAMEtoTAG(pMsg); - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); - } - - /* now parse TAG - that should be present in message from all sources. - * This code is somewhat not compliant with RFC 3164. As of 3164, - * the TAG field is ended by any non-alphanumeric character. In - * practice, however, the TAG often contains dashes and other things, - * which would end the TAG. So it is not desirable. As such, we only - * accept colon and SP to be terminators. Even there is a slight difference: - * a colon is PART of the TAG, while a SP is NOT part of the tag - * (it is CONTENT). Starting 2008-04-04, we have removed the 32 character - * size limit (from RFC3164) on the tag. This had bad effects on existing - * envrionments, as sysklogd didn't obey it either (probably another bug - * in RFC3164...). We now receive the full size, but will modify the - * outputs so that only 32 characters max are used by default. - */ - /* The following code in general is quick & dirty - I need to get - * it going for a test, rgerhards 2004-11-16 */ - /* lol.. we tried to solve it, just to remind ourselfs that 32 octets - * is the max size ;) we need to shuffle the code again... Just for - * the records: the code is currently clean, but we could optimize it! */ - if(!bTAGCharDetected) { - uchar *pszTAG; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) - return 1; - rsCStrSetAllocIncrement(pStrB, 33); - pWork = pBuf; - iCnt = 0; - while(*p2parse && *p2parse != ':' && *p2parse != ' ') { - rsCStrAppendChar(pStrB, *p2parse++); - ++iCnt; - } - if(*p2parse == ':') { - ++p2parse; - rsCStrAppendChar(pStrB, ':'); - } - rsCStrFinish(pStrB); - - rsCStrConvSzStrAndDestruct(pStrB, &pszTAG, 1); - if(pszTAG == NULL) - { /* rger, 2005-11-10: no TAG found - this implies that what - * we have considered to be the HOSTNAME is most probably the - * TAG. We consider it so probable, that we now adjust it - * that way. So we pick up the previously set hostname, assign - * it to tag and use the sender system (from IP stack) as - * the hostname. This situation is the standard case with - * stock BSD syslogd. - */ - dbgprintf("No TAG in message, assuming that HOSTNAME is missing.\n"); - moveHOSTNAMEtoTAG(pMsg); - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); - } else { /* we have a TAG, so we can happily set it ;) */ - MsgAssignTAG(pMsg, pszTAG); - } - } else { - /* we have no TAG, so we ... */ - /*DO NOTHING*/; - } - } else { - /* we enter this code area when the user has instructed rsyslog NOT - * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 - */ - if(!(flags & INTERNAL_MSG)) - { - dbgprintf("HOSTNAME and TAG not parsed by user configuraton.\n"); - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); - } - } - - /* The rest is the actual MSG */ - MsgSetMSG(pMsg, (char*)p2parse); - - ENDfunc - return 0; /* all ok */ -} - - -/* submit a fully created message to the main message queue. The message is - * fully processed and parsed, so no parsing at all happens. This is primarily +/* submit a message to the main message queue. This is primarily * a hook to prevent the need for callers to know about the main message queue - * (which may change in the future as we will probably have multiple rule - * sets and thus queues...). * rgerhards, 2008-02-13 */ rsRetVal submitMsg(msg_t *pMsg) { + qqueue_t *pQueue; + ruleset_t *pRuleset; DEFiRet; ISOBJ_TYPE_assert(pMsg, msg); + pRuleset = MsgGetRuleset(pMsg); + + pQueue = (pRuleset == NULL) ? pMsgQueue : ruleset.GetRulesetQueue(pRuleset); MsgPrepareEnqueue(pMsg); - qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); + qqueueEnqObj(pQueue, pMsg->flowCtlType, (void*) pMsg); RETiRet; } -/* Log a message to the appropriate log files, users, etc. based on - * the priority. - * rgerhards 2004-11-08: actually, this also decodes all but the PRI part. - * rgerhards 2004-11-09: ... but only, if syslogd could properly be initialized - * if not, we use emergency logging to the console and in - * this case, no further decoding happens. - * changed to no longer receive a plain message but a msg object instead. - * rgerhards-2004-11-16: OK, we are now up to another change... This method - * actually needs to PARSE the message. How exactly this needs to happen depends on - * a number of things. Most importantly, it depends on the source. For example, - * locally received messages (SOURCE_UNIXAF) do NOT have a hostname in them. So - * we need to treat them differntly form network-received messages which have. - * Well, actually not all network-received message really have a hostname. We - * can just hope they do, but we can not be sure. So this method tries to find - * whatever can be found in the message and uses that... Obviously, there is some - * potential for misinterpretation, which we simply can not solve under the - * circumstances given. +/* submit multiple messages at once, very similar to submitMsg, just + * for multi_submit_t. All messages need to go into the SAME queue! + * rgerhards, 2009-06-16 */ -void -logmsg(msg_t *pMsg, int flags) +rsRetVal +multiSubmitMsg(multi_submit_t *pMultiSub) { - char *msg; + int i; + qqueue_t *pQueue; + ruleset_t *pRuleset; + DEFiRet; + assert(pMultiSub != NULL); - BEGINfunc - assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - msg = (char*) pMsg->pszUxTradMsg; - dbgprintf("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); - - /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have - * a traditional syslog message or one formatted according to syslog-protocol. - * We need to apply different parsers depending on that. We use the - * -protocol VERSION field for the detection. - */ - if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); - setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, flags) == 1) { - msgDestruct(&pMsg); - return; - } - } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); - setProtocolVersion(pMsg, 0); - if(parseLegacySyslogMsg(pMsg, flags) == 1) { - msgDestruct(&pMsg); - return; - } + if(pMultiSub->nElem == 0) + FINALIZE; + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { + MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); } - /* ---------------------- END PARSING ---------------- */ - - /* now submit the message to the main queue - then we are done */ - pMsg->msgFlags = flags; - MsgPrepareEnqueue(pMsg); - qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); - ENDfunc + pRuleset = MsgGetRuleset(pMultiSub->ppMsgs[0]); + pQueue = (pRuleset == NULL) ? pMsgQueue : ruleset.GetRulesetQueue(pRuleset); + iRet = qqueueMultiEnqObj(pQueue, pMultiSub); + pMultiSub->nElem = 0; + +finalize_it: + RETiRet; } + + static void reapchild() { @@ -1738,7 +775,6 @@ reapchild() DEFFUNC_llExecFunc(flushRptdMsgsActions) { action_t *pAction = (action_t*) pData; - assert(pAction != NULL); BEGINfunc @@ -1747,8 +783,8 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) * the beginn of the llExec(). This makes it slightly less correct, but * in an acceptable way. -- rgerhards, 2008-09-16 */ - if (pAction->f_prevcount && time(NULL) >= REPEATTIME(pAction)) { - dbgprintf("flush %s: repeated %d times, %d sec.\n", + if (pAction->f_prevcount && datetime.GetTime(NULL) >= REPEATTIME(pAction)) { + DBGPRINTF("flush %s: repeated %d times, %d sec.\n", module.GetStateName(pAction->pMod), pAction->f_prevcount, repeatinterval[pAction->f_repeatcount]); actionWriteToAction(pAction); @@ -1761,32 +797,38 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) } -/* This method flushes reapeat messages. +/* This method flushes repeat messages. */ static void doFlushRptdMsgs(void) { - register selector_t *f; - - /* see if we need to flush any "message repeated n times"... - * Note that this interferes with objects running on other threads. - * We are using appropriate locking inside the function to handle that. - */ - for (f = Files; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, flushRptdMsgsActions, NULL); - } + ruleset.IterateAllActions(flushRptdMsgsActions, NULL); } static void debug_switch() { + time_t tTime; + struct tm tp; struct sigaction sigAct; + datetime.GetTime(&tTime); + localtime_r(&tTime, &tp); if(debugging_on == 0) { debugging_on = 1; - dbgprintf("Switching debugging_on to true\n"); + dbgprintf("\n"); + dbgprintf("\n"); + dbgprintf("********************************************************************************\n"); + dbgprintf("Switching debugging_on to true at %2.2d:%2.2d:%2.2d\n", + tp.tm_hour, tp.tm_min, tp.tm_sec); + dbgprintf("********************************************************************************\n"); } else { - dbgprintf("Switching debugging_on to false\n"); + dbgprintf("********************************************************************************\n"); + dbgprintf("Switching debugging_on to false at %2.2d:%2.2d:%2.2d\n", + tp.tm_hour, tp.tm_min, tp.tm_sec); + dbgprintf("********************************************************************************\n"); + dbgprintf("\n"); + dbgprintf("\n"); debugging_on = 0; } @@ -1801,7 +843,7 @@ void legacyOptsEnq(uchar *line) { legacyOptsLL_t *pNew; - pNew = malloc(sizeof(legacyOptsLL_t)); + pNew = MALLOC(sizeof(legacyOptsLL_t)); if(line == NULL) pNew->line = NULL; else @@ -1930,7 +972,7 @@ void legacyOptsParseTCP(char ch, char *arg) * a minimal delay, but it is much cleaner than the approach of doing everything * inside the signal handler. * rgerhards, 2005-10-26 - * Note: we do not call dbgprintf() as this may cause us to block in case something + * Note: we do not call DBGPRINTF() as this may cause us to block in case something * with the threading is wrong. */ static void doDie(int sig) @@ -1938,10 +980,12 @@ static void doDie(int sig) # define MSG1 "DoDie called.\n" # define MSG2 "DoDie called 5 times - unconditional exit\n" static int iRetries = 0; /* debug aid */ + dbgprintf(MSG1); if(Debug) + if(Debug == DEBUG_FULL) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { - if(Debug) + if(Debug == DEBUG_FULL) write(1, MSG2, sizeof(MSG2) - 1); abort(); } @@ -1965,6 +1009,16 @@ freeAllDynMemForTermination(void) } +/* Finalize and destruct all actions. + */ +static inline void +destructAllActions(void) +{ + ruleset.DestructAllActions(); + bHaveMainQueue = 0; // flag that internal messages need to be temporarily stored +} + + /* die() is called when the program shall end. This typically only occurs * during sigterm or during the initialization. * As die() is intended to shutdown rsyslogd, it is @@ -1978,7 +1032,7 @@ die(int sig) { char buf[256]; - dbgprintf("exiting on signal %d\n", sig); + DBGPRINTF("exiting on signal %d\n", sig); /* IMPORTANT: we should close the inputs first, and THEN send our termination * message. If we do it the other way around, logmsgInternal() may block on @@ -1993,11 +1047,12 @@ die(int sig) */ /* close the inputs */ - dbgprintf("Terminating input threads...\n"); + DBGPRINTF("Terminating input threads...\n"); + glbl.SetGlobalInputTermination(); thrdTerminateAll(); /* and THEN send the termination log message (see long comment above) */ - if (sig) { + if(sig && bLogStatusMsgs) { (void) snprintf(buf, sizeof(buf) / sizeof(char), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"]" " exiting on signal %d.", @@ -2007,17 +1062,17 @@ die(int sig) } /* drain queue (if configured so) and stop main queue worker thread pool */ - dbgprintf("Terminating main queue...\n"); + DBGPRINTF("Terminating main queue...\n"); qqueueDestruct(&pMsgQueue); pMsgQueue = NULL; /* Free ressources and close connections. This includes flushing any remaining * repeated msgs. */ - dbgprintf("Terminating outputs...\n"); - freeSelectors(); + DBGPRINTF("Terminating outputs...\n"); + destructAllActions(); - dbgprintf("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); + DBGPRINTF("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); /* rger 2005-02-22 * now clean up the in-memory structures. OK, the OS * would also take care of that, but if we do it @@ -2026,8 +1081,6 @@ die(int sig) */ tplDeleteAll(); - remove_pid(PidFile); - /* de-init some modules */ modExitIminternal(); @@ -2040,22 +1093,18 @@ die(int sig) legacyOptsFree(); + /* destruct our global properties */ + if(pInternalInputName != NULL) + prop.Destruct(&pInternalInputName); + if(pLocalHostIP != NULL) + prop.Destruct(&pLocalHostIP); + /* terminate the remaining classes */ GlobalClassExit(); - /* TODO: this would also be the right place to de-init the builtin output modules. We - * do not currently do that, because the module interface does not allow for - * it. This will come some time later (it's essential with loadable modules). - * For the time being, this is a memory leak on exit, but as the process is - * terminated, we do not really bother about it. - * rgerhards, 2007-08-03 - * I have added some code now, but all that mod init/de-init should be moved to - * init, so that modules are unloaded and reloaded on HUP to. Eventually it should go - * into freeSelectors() - but that needs to be seen. -- rgerhards, 2007-08-09 - */ module.UnloadAndDestructAll(eMOD_LINK_ALL); - dbgprintf("Clean shutdown completed, bye\n"); + DBGPRINTF("Clean shutdown completed, bye\n"); /* dbgClassExit MUST be the last one, because it de-inits the debug system */ dbgClassExit(); @@ -2065,6 +1114,9 @@ die(int sig) */ freeAllDynMemForTermination(); /* NO CODE HERE - feeelAllDynMemForTermination() must be the last thing before exit()! */ + + remove_pid(PidFile); + exit(0); /* "good" exit, this is the terminator function for rsyslog [die()] */ } @@ -2080,7 +1132,7 @@ static void doexit() /* set the maximum message size */ -static rsRetVal setMaxMsgSize(void __attribute__((unused)) *pVal, int iNewVal) +static rsRetVal setMaxMsgSize(void __attribute__((unused)) *pVal, long iNewVal) { return glbl.SetMaxLine(iNewVal); } @@ -2112,7 +1164,10 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) iFiles, errStr, (long) maxFiles.rlim_max); ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); } - dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); +#ifdef USE_UNLIMITED_SELECT + glbl.SetFdSetSize(howmany(iFiles, __NFDBITS) * sizeof (fd_mask)); +#endif + DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); finalize_it: RETiRet; @@ -2123,7 +1178,7 @@ finalize_it: static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { umask(iUmask); - dbgprintf("umask set to 0%3.3o.\n", iUmask); + DBGPRINTF("umask set to 0%3.3o.\n", iUmask); return RS_RET_OK; } @@ -2179,56 +1234,6 @@ static void doDropPrivUid(int iUid) } -/* helper to freeSelectors(), used with llExecFunc() to flush - * pending output. -- rgerhards, 2007-08-02 - * We do not need to lock the action object here as the processing - * queue is already empty and no other threads are running when - * we call this function. -- rgerhards, 2007-12-12 - */ -DEFFUNC_llExecFunc(freeSelectorsActions) -{ - action_t *pAction = (action_t*) pData; - - assert(pAction != NULL); - - /* flush any pending output */ - if(pAction->f_prevcount) { - actionWriteToAction(pAction); - } - - return RS_RET_OK; /* never fails ;) */ -} - - -/* Close all open log files and free selector descriptor array. - */ -static void freeSelectors(void) -{ - selector_t *f; - selector_t *fPrev; - - if(Files != NULL) { - dbgprintf("Freeing log structures.\n"); - - for(f = Files ; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, freeSelectorsActions, NULL); - } - - /* actions flushed and ready for destruction - so do that... */ - f = Files; - while (f != NULL) { - fPrev = f; - f = f->f_next; - selectorDestruct(fPrev); - } - - /* Reflect the deletion of the selectors linked list. */ - Files = NULL; - bHaveMainQueue = 0; - } -} - - /* helper to generateConfigDAG, to print out all actions via * the llExecFunc() facility. * rgerhards, 2007-08-02 @@ -2305,14 +1310,14 @@ DEFFUNC_llExecFunc(generateConfigDAGAction) static rsRetVal generateConfigDAG(uchar *pszDAGFile) { - selector_t *f; + //rule_t *f; FILE *fp; int iActUnit = 1; - int bHasFilter = 0; /* filter associated with this action unit? */ - int bHadFilter; - int i; + //int bHasFilter = 0; /* filter associated with this action unit? */ + //int bHadFilter; + //int i; struct dag_info dagInfo; - char *pszFilterName; + //char *pszFilterName; char szConnectingNode[64]; DEFiRet; @@ -2341,6 +1346,8 @@ generateConfigDAG(uchar *pszDAGFile) strcpy(szConnectingNode, "act0_0"); dagInfo.bDiscarded = 0; +/* TODO: re-enable! */ +#if 0 for(f = Files; f != NULL ; f = f->f_next) { /* BSD-Style filters are currently ignored */ bHadFilter = bHasFilter; @@ -2396,6 +1403,7 @@ generateConfigDAG(uchar *pszDAGFile) ++iActUnit; } +#endif fprintf(fp, "\t%s -> act%d_0\n", szConnectingNode, iActUnit); fprintf(fp, "\tact%d_0\t\t[label=discard shape=box]\n" @@ -2407,20 +1415,6 @@ finalize_it: } -/* helper to dbPrintInitInfo, to print out all actions via - * the llExecFunc() facility. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(dbgPrintInitInfoAction) -{ - DEFiRet; - iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); - - RETiRet; -} - - /* print debug information as part of init(). This pretty much * outputs the whole config of rsyslogd. I've moved this code * out of init() to clean it somewhat up. @@ -2428,47 +1422,8 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) */ static void dbgPrintInitInfo(void) { - selector_t *f; - int iSelNbr = 1; - int i; - - dbgprintf("\nActive selectors:\n"); - for (f = Files; f != NULL ; f = f->f_next) { - dbgprintf("Selector %d:\n", iSelNbr++); - if(f->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); - if(f->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", - f->eHostnameCmpMode == HN_COMP_MATCH ? - "only" : "allbut", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp)); - if(f->f_filter_type == FILTER_PRI) { - for (i = 0; i <= LOG_NFACILITIES; i++) - if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", f->f_filterData.f_pmask[i]); - } else if(f->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); - } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName)); - dbgprintf("\tOperation: "); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue)); - dbgprintf("\tAction...: "); - } - - dbgprintf("\nActions:\n"); - llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - - dbgprintf("\n"); - } - dbgprintf("\n"); + ruleset.DebugPrintAll(); + DBGPRINTF("\n"); if(bDebugPrintTemplateList) tplPrintList(); if(bDebugPrintModuleList) @@ -2478,24 +1433,17 @@ static void dbgPrintInitInfo(void) if(bDebugPrintCfSysLineHandlerList) dbgPrintCfSysLineHandlers(); - dbgprintf("Messages with malicious PTR DNS Records are %sdropped.\n", + DBGPRINTF("Messages with malicious PTR DNS Records are %sdropped.\n", glbl.GetDropMalPTRMsgs() ? "" : "not "); - dbgprintf("Control characters are %sreplaced upon reception.\n", - bEscapeCCOnRcv? "" : "not "); - - if(bEscapeCCOnRcv) - dbgprintf("Control character escape sequence prefix is '%c'.\n", - cCCEscapeChar); - - dbgprintf("Main queue size %d messages.\n", iMainMsgQueueSize); - dbgprintf("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", + DBGPRINTF("Main queue size %d messages.\n", iMainMsgQueueSize); + DBGPRINTF("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", iMainMsgQueueNumWorkers, iMainMsgQtoWrkShutdown, iMainMsgQPersistUpdCnt); - dbgprintf("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", + DBGPRINTF("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", iMainMsgQtoQShutdown, iMainMsgQtoActShutdown, iMainMsgQtoEnq); - dbgprintf("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", + DBGPRINTF("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", iMainMsgQHighWtrMark, iMainMsgQLowWtrMark, iMainMsgQDiscardMark, iMainMsgQDiscardSeverity); - dbgprintf("Main queue save on shutdown %d, max disk space allowed %lld\n", + DBGPRINTF("Main queue save on shutdown %d, max disk space allowed %lld\n", bMainMsgQSaveOnShutdown, iMainMsgQueMaxDiskSpace); /* TODO: add iActionRetryCount = 0; @@ -2506,7 +1454,34 @@ static void dbgPrintInitInfo(void) setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ - dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); + DBGPRINTF("Work Directory: '%s'.\n", glbl.GetWorkDir()); +} + + +/* Actually run the input modules. This happens after privileges are dropped, + * if that is requested. + */ +static rsRetVal +runInputModules(void) +{ + modInfo_t *pMod; + int bNeedsCancel; + + BEGINfunc + /* loop through all modules and activate them (brr...) */ + pMod = module.GetNxtType(NULL, eMOD_IN); + while(pMod != NULL) { + if(pMod->mod.im.bCanRun) { + /* activate here */ + bNeedsCancel = (pMod->isCompatibleWithFeature(sFEATURENonCancelInputTermination) == RS_RET_OK) ? + 0 : 1; + thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun, bNeedsCancel); + } + pMod = module.GetNxtType(pMod, eMOD_IN); + } + + ENDfunc + return RS_RET_OK; /* intentional: we do not care about module errors */ } @@ -2525,11 +1500,10 @@ startInputModules(void) /* loop through all modules and activate them (brr...) */ pMod = module.GetNxtType(NULL, eMOD_IN); while(pMod != NULL) { - if((iRet = pMod->mod.im.willRun()) == RS_RET_OK) { - /* activate here */ - thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun); - } else { - dbgprintf("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); + iRet = pMod->mod.im.willRun(); + pMod->mod.im.bCanRun = (iRet == RS_RET_OK); + if(!pMod->mod.im.bCanRun) { + DBGPRINTF("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); } pMod = module.GetNxtType(pMod, eMOD_IN); } @@ -2539,60 +1513,92 @@ startInputModules(void) } -/* INIT -- Initialize syslogd from configuration table - * init() is called at initial startup AND each time syslogd is HUPed +/* create a main message queue, now also used for ruleset queues. This function + * needs to be moved to some other module, but it is considered acceptable for + * the time being (remember that we want to restructure config processing at large!). + * rgerhards, 2009-10-27 + */ +rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName) +{ + DEFiRet; + + /* switch the message object to threaded operation, if necessary */ + if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { + MsgEnableThreadSafety(); + } + + /* create message queue */ + CHKiRet_Hdlr(qqueueConstruct(ppQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { + /* no queue is fatal, we need to give up in that case... */ + errmsg.LogError(0, iRet, "could not create (ruleset) main message queue"); \ + } + /* name our main queue object (it's not fatal if it fails...) */ + obj.SetName((obj_t*) (*ppQueue), pszQueueName); + + /* ... set some properties ... */ +# define setQPROP(func, directive, data) \ + CHKiRet_Hdlr(func(*ppQueue, data)) { \ + errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ + } +# define setQPROPstr(func, directive, data) \ + CHKiRet_Hdlr(func(*ppQueue, data, (data == NULL)? 0 : strlen((char*) data))) { \ + errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ + } + + setQPROP(qqueueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); + setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); + setQPROP(qqueueSetiDeqBatchSize, "$MainMsgQueueDequeueBatchSize", iMainMsgQueDeqBatchSize); + setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); + setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); + setQPROP(qqueueSetbSyncQueueFiles, "$MainMsgQueueSyncQueueFiles", bMainMsgQSyncQeueFiles); + setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); + setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); + setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); + setQPROP(qqueueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); + setQPROP(qqueueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); + setQPROP(qqueueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); + setQPROP(qqueueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); + setQPROP(qqueueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); + setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); + setQPROP(qqueueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); + setQPROP(qqueueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); + setQPROP(qqueueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); + +# undef setQPROP +# undef setQPROPstr + + /* ... and finally start the queue! */ + CHKiRet_Hdlr(qqueueStart(*ppQueue)) { + /* no queue is fatal, we need to give up in that case... */ + errmsg.LogError(0, iRet, "could not start (ruleset) main message queue"); \ + } + RETiRet; +} + + +/* INIT -- Initialize syslogd * Note that if iConfigVerify is set, only the config file is verified but nothing * else happens. -- rgerhards, 2008-07-28 */ static rsRetVal init(void) { - DEFiRet; rsRetVal localRet; int iNbrActions; + int bHadConfigErr = 0; + ruleset_t *pRuleset; char cbuf[BUFSIZ]; char bufStartUpMsg[512]; struct sigaction sigAct; + DEFiRet; - thrdTerminateAll(); /* stop all running input threads - TODO: reconsider location! */ - - /* initialize some static variables */ - pDfltHostnameCmp = NULL; - pDfltProgNameCmp = NULL; - eDfltHostnameCmpMode = HN_NO_COMP; - - dbgprintf("rsyslog %s - called init()\n", VERSION); - - /* delete the message queue, which also flushes all messages left over */ - if(pMsgQueue != NULL) { - dbgprintf("deleting main message queue\n"); - qqueueDestruct(&pMsgQueue); /* delete pThis here! */ - pMsgQueue = NULL; - } - - /* Close all open log files and free log descriptor array. This also frees - * all output-modules instance data. - */ - freeSelectors(); - - /* Unload all non-static modules */ - dbgprintf("Unloading non-static modules.\n"); - module.UnloadAndDestructAll(eMOD_LINK_DYNAMIC_LOADED); - - dbgprintf("Clearing templates.\n"); - tplDeleteNew(); - - /* re-setting values to defaults (where applicable) */ - /* once we have loadable modules, we must re-visit this code. The reason is - * that config variables are not re-set, because the module is not yet loaded. On - * the other hand, that doesn't matter, because the module got unloaded and is then - * re-loaded, so the variables should be re-set via that way. And this is exactly how - * it works. Loadable module's variables are initialized on load, the rest here. - * rgerhards, 2008-04-28 - */ - conf.cfsysline((uchar*)"ResetConfigVariables"); + DBGPRINTF("rsyslog %s - called init()\n", VERSION); - conf.ReInitConf(); + /* construct the default ruleset */ + ruleset.Construct(&pRuleset); + ruleset.SetName(pRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); + ruleset.ConstructFinalize(pRuleset); /* open the configuration file */ localRet = conf.processConfFile(ConfFile); @@ -2600,12 +1606,14 @@ init(void) if(localRet != RS_RET_OK) { errmsg.LogError(0, localRet, "CONFIG ERROR: could not interpret master config file '%s'.", ConfFile); + bHadConfigErr = 1; } else if(iNbrActions == 0) { errmsg.LogError(0, RS_RET_NO_ACTIONS, "CONFIG ERROR: there are no active actions configured. Inputs will " "run, but no output whatsoever is created."); + bHadConfigErr = 1; } - if(localRet != RS_RET_OK || iNbrActions == 0) { + if((localRet != RS_RET_OK && localRet != RS_RET_NONFATAL_CONFIG_ERR) || iNbrActions == 0) { /* rgerhards: this code is executed to set defaults when the * config file could not be opened. We might think about * abandoning the run in this case - but this, too, is not @@ -2613,38 +1621,27 @@ init(void) * We ignore any errors while doing this - we would be lost anyhow... */ errmsg.LogError(0, NO_ERRCODE, "EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file!"); - selector_t *f = NULL; /* note: we previously used _POSIY_TTY_NAME_MAX+1, but this turned out to be * too low on linux... :-S -- rgerhards, 2008-07-28 */ char szTTYNameBuf[128]; - conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &f); - conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &f); - conf.cfline((uchar*)"*.PANIC\t*", &f); - conf.cfline((uchar*)"syslog.*\troot", &f); + rule_t *pRule = NULL; /* initialization to NULL is *vitally* important! */ + conf.cfline(UCHAR_CONSTANT("*.ERR\t" _PATH_CONSOLE), &pRule); + conf.cfline(UCHAR_CONSTANT("syslog.*\t" _PATH_CONSOLE), &pRule); + conf.cfline(UCHAR_CONSTANT("*.PANIC\t*"), &pRule); + conf.cfline(UCHAR_CONSTANT("syslog.*\troot"), &pRule); if(ttyname_r(0, szTTYNameBuf, sizeof(szTTYNameBuf)) == 0) { snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); - conf.cfline((uchar*)cbuf, &f); + conf.cfline((uchar*)cbuf, &pRule); } else { - dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); + DBGPRINTF("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } - selectorAddList(f); + ruleset.AddRule(ruleset.GetCurrent(), &pRule); } legacyOptsHook(); - /* we are now done with reading the configuration. This is the right time to - * free some objects that were just needed for loading it. rgerhards 2005-10-19 - */ - if(pDfltHostnameCmp != NULL) { - rsCStrDestruct(&pDfltHostnameCmp); - } - - if(pDfltProgNameCmp != NULL) { - rsCStrDestruct(&pDfltProgNameCmp); - } - /* some checks */ if(iMainMsgQueueNumWorkers < 1) { errmsg.LogError(0, NO_ERRCODE, "$MainMsgQueueNumWorkers must be at least 1! Set to 1.\n"); @@ -2672,68 +1669,39 @@ init(void) /* we are done checking the config - now validate if we should actually run or not. * If not, terminate. -- rgerhards, 2008-07-25 */ - if(iConfigVerify) + if(iConfigVerify) { + if(bHadConfigErr) { + /* a bit dirty, but useful... */ + exit(1); + } ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } - /* switch the message object to threaded operation, if necessary */ - if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { - MsgEnableThreadSafety(); + if(bAbortOnUncleanConfig && bHadConfigErr) { + fprintf(stderr, "rsyslogd: $AbortOnUncleanConfig is set, and config is not clean.\n" + "Check error log for details, fix errors and restart. As a last\n" + "resort, you may want to remove $AbortOnUncleanConfig to permit a\n" + "startup with a dirty config.\n"); + exit(2); } /* create message queue */ - CHKiRet_Hdlr(qqueueConstruct(&pMsgQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { + CHKiRet_Hdlr(createMainQueue(&pMsgQueue, UCHAR_CONSTANT("main Q"))) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); exit(1); } - /* name our main queue object (it's not fatal if it fails...) */ - obj.SetName((obj_t*) pMsgQueue, (uchar*) "main queue"); - - /* ... set some properties ... */ -# define setQPROP(func, directive, data) \ - CHKiRet_Hdlr(func(pMsgQueue, data)) { \ - errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ - } -# define setQPROPstr(func, directive, data) \ - CHKiRet_Hdlr(func(pMsgQueue, data, (data == NULL)? 0 : strlen((char*) data))) { \ - errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ - } - - setQPROP(qqueueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); - setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); - setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); - setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); - setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); - setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); - setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); - setQPROP(qqueueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); - setQPROP(qqueueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); - setQPROP(qqueueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); - setQPROP(qqueueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); - setQPROP(qqueueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); - setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); - setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); - setQPROP(qqueueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); - setQPROP(qqueueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); - setQPROP(qqueueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); - -# undef setQPROP -# undef setQPROPstr - - /* ... and finally start the queue! */ - CHKiRet_Hdlr(qqueueStart(pMsgQueue)) { - /* no queue is fatal, we need to give up in that case... */ - fprintf(stderr, "fatal error %d: could not start message queue - rsyslogd can not run!\n", iRet); - exit(1); - } bHaveMainQueue = (MainMsgQueType == QUEUETYPE_DIRECT) ? 0 : 1; - dbgprintf("Main processing queue is initialized and running\n"); + DBGPRINTF("Main processing queue is initialized and running\n"); /* the output part and the queue is now ready to run. So it is a good time - * to start the inputs. Please note that the net code above should be + * to initialize the inputs. Please note that the net code above should be * shuffled to down here once we have everything in input modules. * rgerhards, 2007-12-14 + * NOTE: as of 2009-06-29, the input modules are initialized, but not yet run. + * Keep in mind. though, that the outputs already run if the queue was + * persisted to disk. -- rgerhards */ startInputModules(); @@ -2741,70 +1709,71 @@ init(void) dbgPrintInitInfo(); } - /* we now generate the startup message. It now includes everything to - * identify this instance. -- rgerhards, 2005-08-17 - */ - snprintf(bufStartUpMsg, sizeof(bufStartUpMsg)/sizeof(char), - " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ - "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] restart", - (int) myPid); - logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)bufStartUpMsg, 0); - memset(&sigAct, 0, sizeof (sigAct)); sigemptyset(&sigAct.sa_mask); sigAct.sa_handler = sighup_handler; sigaction(SIGHUP, &sigAct, NULL); - dbgprintf(" (re)started.\n"); + DBGPRINTF(" started.\n"); + + /* we now generate the startup message. It now includes everything to + * identify this instance. -- rgerhards, 2005-08-17 + */ + if(bLogStatusMsgs) { + snprintf(bufStartUpMsg, sizeof(bufStartUpMsg)/sizeof(char), + " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ + "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] start", + (int) myPid); + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)bufStartUpMsg, 0); + } finalize_it: RETiRet; } -/* add a completely-processed selector (after config line parsing) to - * the linked list of selectors. We now need to check - * if it has any actions associated and, if so, link it to the linked - * list. If it has nothing associated with it, we can simply discard - * it. - * We have one special case during initialization: then, the current - * selector is NULL, which means we do not need to care about it at - * all. -- rgerhards, 2007-08-01 +/* Switch the default ruleset (that, what servcies bind to if nothing specific + * is specified). + * rgerhards, 2009-06-12 */ -rsRetVal -selectorAddList(selector_t *f) +static rsRetVal +setDefaultRuleset(void __attribute__((unused)) *pVal, uchar *pszName) { DEFiRet; - int iActionCnt; - static selector_t *nextp = NULL; /* TODO: make this go away (see comment below) */ + CHKiRet(ruleset.SetDefaultRuleset(pszName)); - if(f != NULL) { - CHKiRet(llGetNumElts(&f->llActList, &iActionCnt)); - if(iActionCnt == 0) { - errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); - selectorDestruct(f); - } else { - /* successfully created an entry */ - dbgprintf("selector line successfully processed\n"); - /* TODO: we should use the linked list class for the selector list, else we need to add globals - * ... well nextp could be added temporarily... - * Thanks to varmojfekoj for having the idea to just use "Files" to make this - * code work. I had actually forgotten to fix the code here before moving to 1.18.0. - * And, of course, I also did not migrate the selector_t structure to the linked list class. - * However, that should still be one of the very next things to happen. - * rgerhards, 2007-08-06 - */ - if(Files == NULL) { - Files = f; - } else { - nextp->f_next = f; - } - nextp = f; - } +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + +/* Switch to either an already existing rule set or start a new one. The + * named rule set becomes the new "current" rule set (what means that new + * actions are added to it). + * rgerhards, 2009-06-12 + */ +static rsRetVal +setCurrRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.SetCurrRuleset(pszName); + + if(localRet == RS_RET_NOT_FOUND) { + DBGPRINTF("begin new current rule set '%s'\n", pszName); + CHKiRet(ruleset.Construct(&pRuleset)); + CHKiRet(ruleset.SetName(pRuleset, pszName)); + CHKiRet(ruleset.ConstructFinalize(pRuleset)); + } else { + ABORT_FINALIZE(localRet); } finalize_it: + free(pszName); /* no longer needed */ RETiRet; } @@ -2818,16 +1787,16 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz if (!strcasecmp((char *) pszType, "fixedarray")) { MainMsgQueType = QUEUETYPE_FIXED_ARRAY; - dbgprintf("main message queue type set to FIXED_ARRAY\n"); + DBGPRINTF("main message queue type set to FIXED_ARRAY\n"); } else if (!strcasecmp((char *) pszType, "linkedlist")) { MainMsgQueType = QUEUETYPE_LINKEDLIST; - dbgprintf("main message queue type set to LINKEDLIST\n"); + DBGPRINTF("main message queue type set to LINKEDLIST\n"); } else if (!strcasecmp((char *) pszType, "disk")) { MainMsgQueType = QUEUETYPE_DISK; - dbgprintf("main message queue type set to DISK\n"); + DBGPRINTF("main message queue type set to DISK\n"); } else if (!strcasecmp((char *) pszType, "direct")) { MainMsgQueType = QUEUETYPE_DIRECT; - dbgprintf("main message queue type set to DIRECT (no queueing at all)\n"); + DBGPRINTF("main message queue type set to DIRECT (no queueing at all)\n"); } else { errmsg.LogError(0, RS_RET_INVALID_PARAMS, "unknown mainmessagequeuetype parameter: %s", (char *) pszType); iRet = RS_RET_INVALID_PARAMS; @@ -2856,6 +1825,9 @@ void sighup_handler() sigaction(SIGHUP, &sigAct, NULL); } +void sigttin_handler() +{ +} /* this function pulls all internal messages from the buffer * and puts them into the processing engine. @@ -2863,14 +1835,13 @@ void sighup_handler() * really help us. TODO: add error messages? * rgerhards, 2007-08-03 */ -static void processImInternal(void) +static inline void processImInternal(void) { int iPri; - int iFlags; msg_t *pMsg; - while(iminternalRemoveMsg(&iPri, &pMsg, &iFlags) == RS_RET_OK) { - logmsg(pMsg, iFlags); + while(iminternalRemoveMsg(&iPri, &pMsg) == RS_RET_OK) { + submitMsg(pMsg); } } @@ -2896,25 +1867,18 @@ DEFFUNC_llExecFunc(doHUPActions) static inline void doHUP(void) { - selector_t *f; char buf[512]; - snprintf(buf, sizeof(buf) / sizeof(char), - " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION - "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed, type '%s'.", - (int) myPid, glbl.GetHUPisRestart() ? "restart" : "lightweight"); - errno = 0; - logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)buf, 0); - - if(glbl.GetHUPisRestart()) { - DBGPRINTF("Received SIGHUP, configured to be restart, reloading rsyslogd.\n"); - init(); /* main queue is stopped as part of init() */ - } else { - DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); - for(f = Files; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, doHUPActions, NULL); - } + if(bLogStatusMsgs) { + snprintf(buf, sizeof(buf) / sizeof(char), + " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION + "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed", + (int) myPid); + errno = 0; + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)buf, 0); } + + ruleset.IterateAllActions(doHUPActions, NULL); } @@ -2945,6 +1909,7 @@ mainloop(void) * but a once-a-day wakeup should be quite acceptable. -- rgerhards, 2008-06-09 */ tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/; + //tvSelectTimeout.tv_sec = TIMERINTVL; /* TODO: change this back to the above code when we have a better solution for apc */ tvSelectTimeout.tv_usec = 0; select(1, NULL, NULL, NULL, &tvSelectTimeout); if(bFinished) @@ -2979,49 +1944,11 @@ mainloop(void) bHadHUP = 0; continue; } + // TODO: remove execScheduled(); /* handle Apc calls (if any) */ } ENDfunc } -/* If user is not root, prints warnings or even exits - * TODO: check all dynafiles for write permission - * ... but it is probably better to wait here until we have - * a module interface - rgerhards, 2007-07-23 - */ -static void checkPermissions() -{ -#if 0 - /* TODO: this function must either be redone or removed - now with the input modules, - * there is no such simple check we can do. What we can check, however, is if there is - * any input module active and terminate, if not. -- rgerhards, 2007-12-26 - */ - /* we are not root */ - if (geteuid() != 0) - { - fputs("WARNING: Local messages will not be logged! If you want to log them, run rsyslog as root.\n",stderr); -#ifdef SYSLOG_INET - /* udp enabled and port number less than or equal to 1024 */ - if ( AcceptRemote && (atoi(LogPort) <= 1024) ) - fprintf(stderr, "WARNING: Will not listen on UDP port %s. Use port number higher than 1024 or run rsyslog as root!\n", LogPort); - - /* tcp enabled and port number less or equal to 1024 */ - if( bEnableTCP && (atoi(TCPLstnPort) <= 1024) ) - fprintf(stderr, "WARNING: Will not listen on TCP port %s. Use port number higher than 1024 or run rsyslog as root!\n", TCPLstnPort); - - /* Neither explicit high UDP port nor explicit high TCP port. - * It is useless to run anymore */ - if( !(AcceptRemote && (atoi(LogPort) > 1024)) && !( bEnableTCP && (atoi(TCPLstnPort) > 1024)) ) - { -#endif - fprintf(stderr, "ERROR: Nothing to log, no reason to run. Please run rsyslog as root.\n"); - exit(EXIT_FAILURE); -#ifdef SYSLOG_INET - } -#endif - } -#endif -} - /* load build-in modules * very first version begun on 2007-07-23 by rgerhards @@ -3030,23 +1957,26 @@ static rsRetVal loadBuildInModules(void) { DEFiRet; - if((iRet = module.doModInit(modInitFile, (uchar*) "builtin-file", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitFile, UCHAR_CONSTANT("builtin-file"), NULL)) != RS_RET_OK) { + RETiRet; + } + if((iRet = module.doModInit(modInitPipe, UCHAR_CONSTANT("builtin-pipe"), NULL)) != RS_RET_OK) { RETiRet; } #ifdef SYSLOG_INET - if((iRet = module.doModInit(modInitFwd, (uchar*) "builtin-fwd", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitFwd, UCHAR_CONSTANT("builtin-fwd"), NULL)) != RS_RET_OK) { RETiRet; } #endif - if((iRet = module.doModInit(modInitShell, (uchar*) "builtin-shell", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitShell, UCHAR_CONSTANT("builtin-shell"), NULL)) != RS_RET_OK) { RETiRet; } - if((iRet = module.doModInit(modInitDiscard, (uchar*) "builtin-discard", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitDiscard, UCHAR_CONSTANT("builtin-discard"), NULL)) != RS_RET_OK) { RETiRet; } /* dirty, but this must be for the time being: the usrmsg module must always be - * loaded as last module. This is because it processes any time of action selector. + * loaded as last module. This is because it processes any type of action selector. * If we load it before other modules, these others will never have a chance of * working with the config file. We may change that implementation so that a user name * must start with an alnum, that would definitely help (but would it break backwards @@ -3054,8 +1984,15 @@ static rsRetVal loadBuildInModules(void) * User names now must begin with: * [a-zA-Z0-9_.] */ - if((iRet = module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)) != RS_RET_OK) - RETiRet; + CHKiRet(module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)); + + /* load build-in parser modules */ + CHKiRet(module.doModInit(modInitpmrfc5424, UCHAR_CONSTANT("builtin-pmrfc5424"), NULL)); + CHKiRet(module.doModInit(modInitpmrfc3164, UCHAR_CONSTANT("builtin-pmrfc3164"), NULL)); + + /* and set default parser modules (order is *very* important, legacy (3164) parse needs to go last! */ + CHKiRet(parser.AddDfltParser(UCHAR_CONSTANT("rsyslog.rfc5424"))); + CHKiRet(parser.AddDfltParser(UCHAR_CONSTANT("rsyslog.rfc3164"))); /* ok, initialization of the command handler probably does not 100% belong right in * this space here. However, with the current design, this is actually quite a good @@ -3064,7 +2001,10 @@ static rsRetVal loadBuildInModules(void) * is that rsyslog will terminate if we can not register our built-in config commands. * This, I think, is the right thing to do. -- rgerhards, 2007-07-31 */ + CHKiRet(regCfSysLineHdlr((uchar *)"logrsyslogstatusmessages", 0, eCmdHdlrBinary, NULL, &bLogStatusMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeretrycount", 0, eCmdHdlrInt, NULL, &glbliActionResumeRetryCount, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"defaultruleset", 0, eCmdHdlrGetWord, setDefaultRuleset, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, setCurrRuleset, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszMainMsgQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesize", 0, eCmdHdlrInt, NULL, &iMainMsgQueueSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iMainMsgQHighWtrMark, NULL)); @@ -3072,6 +2012,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardmark", 0, eCmdHdlrInt, NULL, &iMainMsgQDiscardMark, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardseverity", 0, eCmdHdlrSeverity, NULL, &iMainMsgQDiscardSeverity, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iMainMsgQPersistUpdCnt, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bMainMsgQSyncQeueFiles, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetype", 0, eCmdHdlrGetWord, setMainMsgQueType, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iMainMsgQueueNumWorkers, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iMainMsgQtoQShutdown, NULL)); @@ -3081,17 +2022,15 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeueslowdown", 0, eCmdHdlrInt, NULL, &iMainMsgQDeqSlowdown, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreadminimummessages", 0, eCmdHdlrInt, NULL, &iMainMsgQWrkMinMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuemaxfilesize", 0, eCmdHdlrSize, NULL, &iMainMsgQueMaxFileSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuebatchsize", 0, eCmdHdlrSize, NULL, &iMainMsgQueDeqBatchSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iMainMsgQueMaxDiskSpace, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesaveonshutdown", 0, eCmdHdlrBinary, NULL, &bMainMsgQSaveOnShutdown, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuetimebegin", 0, eCmdHdlrInt, NULL, &iMainMsgQueueDeqtWinFromHr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iMainMsgQueueDeqtWinToHr, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"abortonuncleanconfig", 0, eCmdHdlrBinary, NULL, &bAbortOnUncleanConfig, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgreduction", 0, eCmdHdlrBinary, NULL, &bReduceRepeatMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlywhenpreviousissuspended", 0, eCmdHdlrBinary, NULL, &bActExecWhenPrevSusp, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyonceeveryinterval", 0, eCmdHdlrInt, NULL, &iActExecOnceInterval, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeinterval", 0, eCmdHdlrInt, setActionResumeInterval, NULL, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"template", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_TEMPLATE, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"outchannel", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_OUTCHANNEL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"allowedsender", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_ALLOWEDSENDER, NULL)); @@ -3113,11 +2052,6 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroupid", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); - /* now add other modules handlers (we should work on that to be able to do it in ClassInit(), but so far - * that is not possible). -- rgerhards, 2008-01-28 - */ - CHKiRet(actionAddCfSysLineHdrl()); - finalize_it: RETiRet; } @@ -3206,8 +2140,9 @@ static rsRetVal mainThread() CHKiRet(init()); if(Debug && debugging_on) { - dbgprintf("Debugging enabled, SIGUSR1 to turn off debugging.\n"); + DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n"); } + /* Send a signal to the parent so it can terminate. */ if(myPid != ppid) @@ -3215,7 +2150,7 @@ static rsRetVal mainThread() /* If instructed to do so, we now drop privileges. Note that this is not 100% secure, - * because inputs and outputs are already running at this time. However, we can implement + * because outputs are already running at this time. However, we can implement * dropping of privileges rather quickly and it will work in many cases. While it is not * the ultimate solution, the current one is still much better than not being able to * drop privileges at all. Doing it correctly, requires a change in architecture, which @@ -3223,22 +2158,31 @@ static rsRetVal mainThread() */ if(gidDropPriv != 0) { doDropPrivGid(gidDropPriv); - glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ } if(uidDropPriv != 0) { doDropPrivUid(uidDropPriv); - glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ } + /* finally let the inputs run... */ + runInputModules(); /* END OF INTIALIZATION - * ... but keep in mind that we might do a restart and thus init() might - * be called again. If that happens, we must shut down the worker thread, - * do the init() and then restart things. - * rgerhards, 2005-10-24 */ - dbgprintf("initialization completed, transitioning to regular run mode\n"); + DBGPRINTF("initialization completed, transitioning to regular run mode\n"); + + /* close stderr and stdout if they are kept open during a fork. Note that this + * may introduce subtle security issues: if we are in a jail, one may break out of + * it via these descriptors. But if I close them earlier, error messages will (once + * again) not be emitted to the user that starts the daemon. As root jail support + * is still in its infancy (and not really done), we currently accept this issue. + * rgerhards, 2009-06-29 + */ + if(!(Debug == DEBUG_FULL || NoFork)) { + close(1); + close(2); + bErrMsgToStderr = 0; + } mainloop(); @@ -3269,24 +2213,26 @@ InitGlobalClasses(void) CHKiRet(objUse(errmsg, CORE_COMPONENT)); pErrObj = "module"; CHKiRet(objUse(module, CORE_COMPONENT)); - pErrObj = "var"; - CHKiRet(objUse(var, CORE_COMPONENT)); pErrObj = "datetime"; CHKiRet(objUse(datetime, CORE_COMPONENT)); - pErrObj = "vm"; - CHKiRet(objUse(vm, CORE_COMPONENT)); pErrObj = "expr"; CHKiRet(objUse(expr, CORE_COMPONENT)); + pErrObj = "rule"; + CHKiRet(objUse(rule, CORE_COMPONENT)); + pErrObj = "ruleset"; + CHKiRet(objUse(ruleset, CORE_COMPONENT)); pErrObj = "conf"; CHKiRet(objUse(conf, CORE_COMPONENT)); + pErrObj = "prop"; + CHKiRet(objUse(prop, CORE_COMPONENT)); + pErrObj = "parser"; + CHKiRet(objUse(parser, CORE_COMPONENT)); /* intialize some dummy classes that are not part of the runtime */ pErrObj = "action"; CHKiRet(actionClassInit()); pErrObj = "template"; CHKiRet(templateInit()); - pErrObj = "parser"; - CHKiRet(parserClassInit()); /* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ pErrObj = "net"; @@ -3319,34 +2265,16 @@ GlobalClassExit(void) /* first, release everything we used ourself */ objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ + objRelease(prop, CORE_COMPONENT); objRelease(conf, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); + objRelease(rule, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ - objRelease(vm, CORE_COMPONENT); - objRelease(var, CORE_COMPONENT); + parserClassExit(); /* this is hack, currently core_modules do not get this automatically called */ objRelease(datetime, CORE_COMPONENT); /* TODO: implement the rest of the deinit */ -#if 0 - CHKiRet(datetimeClassInit(NULL)); - CHKiRet(msgClassInit(NULL)); - CHKiRet(strmClassInit(NULL)); - CHKiRet(wtiClassInit(NULL)); - CHKiRet(wtpClassInit(NULL)); - CHKiRet(qqueueClassInit(NULL)); - CHKiRet(vmstkClassInit(NULL)); - CHKiRet(sysvarClassInit(NULL)); - CHKiRet(vmClassInit(NULL)); - CHKiRet(vmopClassInit(NULL)); - CHKiRet(vmprgClassInit(NULL)); - CHKiRet(ctok_tokenClassInit(NULL)); - CHKiRet(ctokClassInit(NULL)); - CHKiRet(exprClassInit(NULL)); - - /* dummy "classes" */ - CHKiRet(actionClassInit()); - CHKiRet(templateInit()); -#endif /* dummy "classes */ strExit(); @@ -3384,7 +2312,7 @@ bufOptAdd(char opt, char *arg) DEFiRet; bufOpt_t *pBuf; - if((pBuf = malloc(sizeof(bufOpt_t))) == NULL) + if((pBuf = MALLOC(sizeof(bufOpt_t))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pBuf->optchar = opt; @@ -3440,12 +2368,11 @@ doGlblProcessInit(void) int i; DEFiRet; - checkPermissions(); thrdInit(); - if( !(Debug || NoFork) ) + if( !(Debug == DEBUG_FULL || NoFork) ) { - dbgprintf("Checking pidfile.\n"); + DBGPRINTF("Checking pidfile.\n"); if (!check_pid(PidFile)) { memset(&sigAct, 0, sizeof (sigAct)); @@ -3467,7 +2394,9 @@ doGlblProcessInit(void) exit(1); /* "good" exit - after forking, not diasabling anything */ } num_fds = getdtablesize(); - for (i= 0; i < num_fds; i++) + close(0); + /* we keep stdout and stderr open in case we have to emit something */ + for (i = 3; i < num_fds; i++) (void) close(i); untty(); } @@ -3476,12 +2405,10 @@ doGlblProcessInit(void) fputs(" Already running.\n", stderr); exit(1); /* "good" exit, done if syslogd is already running */ } - } else { - debugging_on = 1; } /* tuck my process id away */ - dbgprintf("Writing pidfile %s.\n", PidFile); + DBGPRINTF("Writing pidfile %s.\n", PidFile); if (!check_pid(PidFile)) { if (!write_pid(PidFile)) @@ -3513,6 +2440,8 @@ doGlblProcessInit(void) sigaction(SIGCHLD, &sigAct, NULL); sigAct.sa_handler = Debug ? debug_switch : SIG_IGN; sigaction(SIGUSR1, &sigAct, NULL); + sigAct.sa_handler = sigttin_handler; + sigaction(SIGTTIN, &sigAct, NULL); /* (ab)used to interrupt input threads */ sigAct.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sigAct, NULL); sigaction(SIGXFSZ, &sigAct, NULL); /* do not abort if 2gig file limit is hit */ @@ -3585,6 +2514,7 @@ int realMain(int argc, char **argv) iCompatibilityMode = atoi(optarg); break; case 'd': /* debug - must be handled now, so that debug is active during init! */ + debugging_on = 1; Debug = 1; break; case 'e': /* log every message (no repeat message supression) */ @@ -3627,7 +2557,7 @@ int realMain(int argc, char **argv) if ((argc -= optind)) usage(); - dbgprintf("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", + DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", VERSION, iCompatibilityMode, glblModPath == NULL ? "" : (char*)glblModPath); /* we are done with the initial option parsing and processing. Now we init the system. */ @@ -3644,6 +2574,15 @@ int realMain(int argc, char **argv) /* doing some core initializations */ + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInternalInputName)); + CHKiRet(prop.SetString(pInternalInputName, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslgod") - 1)); + CHKiRet(prop.ConstructFinalize(pInternalInputName)); + + CHKiRet(prop.Construct(&pLocalHostIP)); + CHKiRet(prop.SetString(pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pLocalHostIP)); + /* get our host and domain names - we need to do this early as we may emit * error log messages, which need the correct hostname. -- rgerhards, 2008-04-04 */ @@ -3692,6 +2631,7 @@ int realMain(int argc, char **argv) */ glbl.SetLocalHostName(LocalHostName); glbl.SetLocalDomain(LocalDomain); + glbl.GenerateLocalHostNameProperty(); /* must be redone after conf processing, FQDN setting may have changed */ /* initialize the objects */ if((iRet = modInitIminternal()) != RS_RET_OK) { @@ -3709,7 +2649,7 @@ int realMain(int argc, char **argv) /* END core initializations - we now come back to carrying out command line options*/ while((iRet = bufOptRemove(&ch, &arg)) == RS_RET_OK) { - dbgprintf("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); + DBGPRINTF("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); switch((char)ch) { case '4': glbl.SetDefPFFamily(PF_INET); @@ -3793,11 +2733,12 @@ int realMain(int argc, char **argv) } else { fprintf(stderr, "error -p is no longer supported, use module imuxsock instead"); } + break; case 'q': /* add hostname if DNS resolving has failed */ - *net.pACLAddHostnameOnFail = 1; + *(net.pACLAddHostnameOnFail) = 1; break; case 'Q': /* dont resolve hostnames in ACL to IPs */ - *net.pACLDontResolve = 1; + *(net.pACLDontResolve) = 1; break; case 'r': /* accept remote messages */ if(iCompatibilityMode < 3) { @@ -3829,7 +2770,7 @@ int realMain(int argc, char **argv) case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); if(iHelperUOpt & 0x01) - bParseHOSTNAMEandTAG = 0; + glbl.SetParseHOSTNAMEandTAG(0); if(iHelperUOpt & 0x02) bChDirRoot = 0; break; @@ -3888,6 +2829,9 @@ int realMain(int argc, char **argv) if(!iConfigVerify) CHKiRet(doGlblProcessInit()); + /* re-generate local host name property, as the config may have changed our FQDN settings */ + glbl.GenerateLocalHostNameProperty(); + CHKiRet(mainThread()); /* do any de-init's that need to be done AFTER this comment */ diff --git a/tools/syslogd.h b/tools/syslogd.h index 8b9bd131..c3b99f9d 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -29,70 +29,9 @@ #include "linkedlist.h" #include "expr.h" - -#ifndef _PATH_CONSOLE -#define _PATH_CONSOLE "/dev/console" -#endif - - -/* This structure represents the files that will have log - * copies printed. - * RGerhards 2004-11-08: Each instance of the filed structure - * describes what I call an "output channel". This is important - * to mention as we now allow database connections to be - * present in the filed structure. If helps immensely, if we - * think of it as the abstraction of an output channel. - * rgerhards, 2005-10-26: The structure below provides ample - * opportunity for non-thread-safety. Each of the variable - * accesses must be carefully evaluated, many of them probably - * be guarded by mutexes. But beware of deadlocks... - * rgerhards, 2007-08-01: as you can see, the structure has shrunk pretty much. I will - * remove some of the comments some time. It's still the structure that controls much - * of the processing that goes on in syslogd, but it now has lots of helpers. - */ -struct filed { - struct filed *f_next; /* next in linked list */ - /* filter properties */ - enum { - FILTER_PRI = 0, /* traditional PRI based filer */ - FILTER_PROP = 1, /* extended filter, property based */ - FILTER_EXPR = 2 /* extended filter, expression based */ - } f_filter_type; - EHostnameCmpMode eHostnameCmpMode; - cstr_t *pCSHostnameComp; /* hostname to check */ - cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ - union { - u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ - struct { - cstr_t *pCSPropName; - enum { - FIOP_NOP = 0, /* do not use - No Operation */ - FIOP_CONTAINS = 1, /* contains string? */ - FIOP_ISEQUAL = 2, /* is (exactly) equal? */ - FIOP_STARTSWITH = 3, /* starts with a string? */ - FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ - FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ - } operation; - regex_t *regex_cache; /* cache for compiled REs, if such are used */ - cstr_t *pCSCompValue; /* value to "compare" against */ - char isNegated; /* actually a boolean ;) */ - } prop; - expr_t *f_expr; /* expression object */ - } f_filterData; - - linkedList_t llActList; /* list of configured actions */ -}; - - -#include "net.h" /* TODO: remove when you remoe isAllowedSender from here! */ -void untty(void); -rsRetVal selectorConstruct(selector_t **ppThis); -rsRetVal selectorDestruct(void *pVal); -rsRetVal selectorAddList(selector_t *f); /* the following prototypes should go away once we have an input * module interface -- rgerhards, 2007-12-12 */ -void logmsg(msg_t *pMsg, int flags); extern int NoHops; extern int send_to_all; extern int Debug; diff --git a/tools/zpipe.c b/tools/zpipe.c new file mode 100644 index 00000000..bde6c5c1 --- /dev/null +++ b/tools/zpipe.c @@ -0,0 +1,254 @@ +/* zpipe.c: example of proper use of zlib's inflate() and deflate() + Not copyrighted -- provided to the public domain + Version 1.5 11 December 2005 Mark Adler + Version 2.0 03 June 2009 Rainer Gerhards */ + +/* RSYSLOG NOTE: + * This file is beeing distributed as part of rsyslog, but is just an + * add-on. Most importantly, rsyslog's copyright does not apply but + * rather the (non-) copyright stated above. + */ + +/* Version history: + 1.0 30 Oct 2004 First version + 1.1 8 Nov 2004 Add void casting for unused return values + Use switch statement for inflate() return values + 1.2 9 Nov 2004 Add assertions to document zlib guarantees + 1.3 6 Apr 2005 Remove incorrect assertion in inf() + 1.4 11 Dec 2005 Add hack to avoid MSDOS end-of-line conversions + Avoid some compiler warnings for input and output buffers + 2.0 03 Jun 2009 Add hack to support multiple deflate records inside a single + file on inflate. This is needed in order to support reading + files created by rsyslog's zip output writer. + */ + +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include "zlib.h" + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include <fcntl.h> +# include <io.h> +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 + +/* Compress from file source to file dest until EOF on source. + def() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_STREAM_ERROR if an invalid compression + level is supplied, Z_VERSION_ERROR if the version of zlib.h and the + version of the library linked do not match, or Z_ERRNO if there is + an error reading or writing the files. */ +int def(FILE *source, FILE *dest, int level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, level); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + flush = feof(source) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + return Z_OK; +} + + +/* initialize stream for deflating (we need this in case of + * multiple records. + * rgerhards, 2009-06-03 + */ +int doInflateInit(z_stream *strm) +{ + int ret; + + /* allocate inflate state */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + strm->avail_in = 0; + strm->next_in = Z_NULL; + ret = inflateInit(strm); + return ret; +} + + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ +int inf(FILE *source, FILE *dest) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + int len; + unsigned char *next_in_save; + unsigned char out[CHUNK]; + + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + len = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + if (len == 0) { + break; + } + strm.avail_in = len; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + strm.avail_out = CHUNK; + strm.next_out = out; + do { + /* fprintf(stderr, "---inner LOOP---, avail_in %d, avail_out %d Byte 0: %x, 1: %x\n", strm.avail_in, strm.avail_out, *strm.next_in, *(strm.next_in+1));*/ + do { + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + /* handle the case that more than one deflate record is contained + * in a single file. -- rgerhards, 2009-06-03 + */ + if(ret == Z_STREAM_END) { + len -= strm.total_in; + if(len > 0) { + next_in_save = strm.next_in; + (void)inflateEnd(&strm); + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + strm.avail_in = len; + strm.next_in = next_in_save; + strm.avail_out = CHUNK; + strm.next_out = out; + ret = Z_OK; /* continue outer loop */ + } + } + } while (strm.avail_in > 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + +/* report a zlib or i/o error */ +void zerr(int ret) +{ + fputs("zpipe: ", stdout); + switch (ret) { + case Z_ERRNO: + if (ferror(stdin)) + fputs("error reading stdin\n", stdout); + if (ferror(stdout)) + fputs("error writing stdout\n", stdout); + break; + case Z_STREAM_ERROR: + fputs("invalid compression level\n", stdout); + break; + case Z_DATA_ERROR: + fputs("invalid or incomplete deflate data\n", stdout); + break; + case Z_MEM_ERROR: + fputs("out of memory\n", stdout); + break; + case Z_VERSION_ERROR: + fputs("zlib version mismatch!\n", stdout); + } +} + +/* compress or decompress from stdin to stdout */ +int main(int argc, char **argv) +{ + int ret; + + /* avoid end-of-line conversions */ + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + + /* do compression if no arguments */ + if (argc == 1) { + ret = def(stdin, stdout, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* do decompression if -d specified */ + else if (argc == 2 && strcmp(argv[1], "-d") == 0) { + ret = inf(stdin, stdout); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* otherwise, report usage */ + else { + fputs("zpipe usage: zpipe [-d] < source > dest\n", stdout); + return 1; + } +} |
