diff options
| -rw-r--r-- | ChangeLog | 19 | ||||
| -rw-r--r-- | doc/imjournal.html | 68 | ||||
| -rw-r--r-- | doc/imrelp.html | 6 | ||||
| -rw-r--r-- | doc/omfile.html | 8 | ||||
| -rw-r--r-- | doc/omjournal.html | 3 | ||||
| -rwxr-xr-x | plugins/imjournal/imjournal.c | 79 | ||||
| -rw-r--r-- | plugins/imzmq3/imzmq3.c | 70 | ||||
| -rw-r--r-- | rsyslog.service.in | 3 | ||||
| -rw-r--r-- | runtime/ratelimit.c | 8 | ||||
| -rw-r--r-- | tools/syslogd.c | 2 |
10 files changed, 219 insertions, 47 deletions
@@ -1,4 +1,23 @@ --------------------------------------------------------------------------- +Version 7.4.1 [v7.4-stable] 2013-06-?? +- imjournal: add ratelimiting capability + The original imjournal code did not support ratelimiting at all. We + now have our own ratelimiter. This can mitigate against journal + database corruption, when the journal re-sends old data. This is a + current bug in systemd journal, but we won't outrule this to happen + in the future again. So it is better to have a safeguard in place. + By default, we permit 20,000 messages witin 10 minutes. This may + be a bit restrictive, but given the risk potential it seems reasonable. + Users requiring larger traffic flows can always adjust the value. +- bugfix: potential loop in rate limiting + if the message that tells about rate-limiting gets rate-limited itself, + it will potentially create and endless loop +- bugfix: potential segfault in imjournal if journal DB is corrupted +- bugfix: prevent a segfault in imjournal if state file is not defined +- bugfix imzmq3: potential segfault on startup + if no problem happend at startup, everything went fine + Thanks to Hongfei Cheng and Brian Knox for the patch +--------------------------------------------------------------------------- Version 7.4.0 [v7.4-stable] 2013-06-06 This starts a new stable branch based on 7.3.15 plus the following changes: - add --enable-cached-man-pages ./configure option diff --git a/doc/imjournal.html b/doc/imjournal.html index dbf9279e..5a18d5d6 100644 --- a/doc/imjournal.html +++ b/doc/imjournal.html @@ -1,6 +1,6 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head> -<meta http-equiv="Content-Language" content="en"><title>Text File Input Monitor</title></head> +<meta http-equiv="Content-Language" content="en"><title>Systemd Journal Input Module</title></head> <body> <a href="rsyslog_conf_modules.html">back</a> @@ -11,28 +11,84 @@ <p><b>Description</b>:</p> <p>Provides the ability to import structured log messages from systemd journal to syslog.</p> +<p>Note that this module reads the journal database, what is considered a +relativly performance-intense operation. As such, the performance of a +configuration utilizing this +module may be notably slower then when using +<a href="imuxsock.html">imuxsock</a>. The journal provides imuxsock with a +copy of all "classical" syslog messages, however, it does not provide +structured data. If the latter is needed, imjournal must be used. Otherwise, +imjournal may be simply replaced by imuxsock. +<p>We suggest to check out our short presentation on +<a href="http://youtu.be/GTS7EuSdFKE">rsyslog journal integration</a> to +learn more details of anticipated use cases. + +<p><b>Warning:</b> Some versions of systemd journal have problems with database +corruption, which leads to the journal to return the same data endlessly +in a thight loop. This results in massive message duplication inside rsyslog +probably resulting in a denial-of-service when the system ressouces get +exhausted. This can be somewhat mitigated by using proper rate-limiters, but +even then there are spikes of old data which are endlessly repeated. By default, +ratelimiting is activated and permits to process 20,000 messages within 10 +seconds, what should be well enough for most use cases. If insufficient, use +the parameters described below to adjust the permitted volume. +<b>It is strongly recommended to use this plugin only if there +is hard need to do so.</b> <p><b>Configuration Directives</b>:</p> <p><b>Module Directives</b></p> <ul> -<li><span style="font-weight: bold;">PersistStateInterval -messages</span><br> +<li><b>PersistStateInterval</b> messages<br> This is a global setting. It specifies how often should the journal state be persisted. This option is useful for rsyslog to start reding from the last journal message it read. -<li><span style="font-weight: bold;">StateFile -/path/to/file</span><br> +<li><b>StateFile</b> /path/to/file<br> This is a global setting. It specifies where the state file for persisting journal state is located. + +<li><b>ratelimit.interval</b> seconds (default: 600)<br> +Specifies the interval in seconds onto which rate-limiting is to be applied. +If more than ratelimit.burst messages are read during that interval, further +messages up to the end of the interval are discarded. The number of messages +discarded is emitted at the end of the interval (if there were any discards). +<br>Setting this to value zero turns off ratelimiting. Note that it is +<b>not recommended to turn of ratelimiting</b>, except that you know for +sure journal database entries will never be corrupted. Without ratelimiting, +a corrupted systemd journal database may cause a kind of denial of service (we +are stressing this point as multiple users have reported us such problems +with the journal database - information current as of June 2013). + +<li><b>ratelimit.burst</b> messages (default: 20000)<br> +Specifies the maximum number of messages that can be emitted within the +ratelimit.interval interval. For futher information, see description there. + +</ul> + +<p><b>Legacy Configuration Directives</b>:</p> +<ul> +<li>$imjournalPersistStateInterval <Delimiter><br> +Equivalent to: ratelimit.PersistStateInterval</li> +<li>$imjournalStateFile <Delimiter><br> +Equivalent to: ratelimit.StateFile</li> +<li>$imjournalRatelimitInterval <Delimiter><br> +Equivalent to: ratelimit.interval</li> +<li>$imjournalRatelimitBurst <Delimiter><br> +Equivalent to: ratelimit.burst</li> </ul> + <b>Caveats/Known Bugs:</b> <p> +<ul> +<li>As stated above, a corrupted systemd journal database can cause major +problems, depending on what the corruption results in. This is beyond the +control of the rsyslog team. +</ul> </p> <p><b>Sample:</b></p> <p> The following example shows pulling structured imjournal messages and saving them into /var/log/ceelog </p> -<textarea rows="15" cols="60"> +<textarea rows="11" cols="60"> module(load="imjournal" PersistStateInterval="100" StateFile="/path/to/file") #load imjournal module module(load="mmjsonparse") #load mmjsonparse module for structured logs diff --git a/doc/imrelp.html b/doc/imrelp.html index 9f3e4875..f7fcc4b3 100644 --- a/doc/imrelp.html +++ b/doc/imrelp.html @@ -30,14 +30,12 @@ Clients send messages to the RELP server via omrelp.</p> <p><b>Configuration Directives</b>:</p> <ul> -<li><b>Ruleset</b> <name></br> -Binds the specified ruleset to all RELP listeners. <li><b>Port</b> <port><br> Starts a RELP server on selected port</li> </ul> <b>Caveats/Known Bugs:</b> <ul> -<li>see description</li> +<li>ruleset can only be bound via legacy configuration format</li> <li>To obtain the remote system's IP address, you need to have at least librelp 1.0.0 installed. Versions below it return the hostname instead of the IP address.</li> @@ -54,7 +52,7 @@ input(type="imrelp" port="20514") <p><b>Legacy Configuration Directives</b>:</p> <ul> <li>InputRELPServerBindRuleset <name> (available in 6.3.6+)</br> -equivalent to: RuleSet +Binds the specified ruleset to all RELP listeners. <li>InputRELPServerRun <port><br> equivalent to: Port</li> </ul> diff --git a/doc/omfile.html b/doc/omfile.html index a5c3bb3b..438d694c 100644 --- a/doc/omfile.html +++ b/doc/omfile.html @@ -111,8 +111,12 @@ File="/var/log/messages") <p><b>Legacy Configuration Directives</b>:</p> <ul> - <li><strong>$DynaFileCacheSize </strong>(not mandatory, default will be used)<br> - Defines a template to be used for the output. <br></li><br> + <li><strong>$DynaFileCacheSize </strong>(not mandatory, default 10)<br> + defines the maximum size of the dynafile cache for <b>this</b> + action. Cache size greatly affects performance and should be + set so that it matches the actual need. Note that files + inside the cache are kept open until either rsyslogd + is HUPed or the file is evicted from the cache.<br></li><br> <li><strong>$OMFileZipLevel </strong>0..9 [default 0]<br> if greater 0, turns on gzip compression of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.<br></li><br> diff --git a/doc/omjournal.html b/doc/omjournal.html index c42d9841..6124e40c 100644 --- a/doc/omjournal.html +++ b/doc/omjournal.html @@ -19,6 +19,9 @@ and processed by its tools. <p>A typical use case we had on our mind is a SOHO environment, where the user wants to include syslog data obtained from the local router to be part of the journal data. +<p>We suggest to check out our short presentation on +<a href="http://youtu.be/GTS7EuSdFKE">rsyslog journal integration</a> to +learn more details of anticipated use cases. <p> </p> <p><b>Module Configuration Parameters</b>:</p> diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c index 8f1824c9..74ef7115 100755 --- a/plugins/imjournal/imjournal.c +++ b/plugins/imjournal/imjournal.c @@ -3,7 +3,7 @@ * To test under Linux: * emmit log message into systemd journal * - * Copyright (C) 2008-2012 Adiscon GmbH + * Copyright (C) 2008-2013 Adiscon GmbH * * This file is part of rsyslog. * @@ -33,6 +33,7 @@ #include <sys/poll.h> #include <sys/socket.h> #include <errno.h> +#include <systemd/sd-journal.h> #include "dirty.h" #include "cfsysline.h" @@ -47,7 +48,7 @@ #include "errmsg.h" #include "srUtils.h" #include "unicode-helper.h" -#include <systemd/sd-journal.h> +#include "ratelimit.h" MODULE_TYPE_INPUT MODULE_TYPE_NOKEEP @@ -64,11 +65,15 @@ DEFobjCurrIf(errmsg) static struct configSettings_s { char *stateFile; int iPersistStateInterval; + int ratelimitInterval; + int ratelimitBurst; } cs; -/* module-gloval parameters */ +/* module-global parameters */ static struct cnfparamdescr modpdescr[] = { { "statefile", eCmdHdlrGetWord, 0 }, + { "ratelimit.interval", eCmdHdlrInt, 0 }, + { "ratelimit.burst", eCmdHdlrInt, 0 }, { "persiststateinterval", eCmdHdlrInt, 0 } }; static struct cnfparamblk modpblk = @@ -84,6 +89,7 @@ static int bLegacyCnfModGlobalsPermitted = 1;/* are legacy module-global config static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this module */ static prop_t *pLocalHostIP = NULL; /* a pseudo-constant propterty for 127.0.0.1 */ +static ratelimit_t *ratelimiter = NULL; static sd_journal *j; /* enqueue the the journal message into the message queue. @@ -121,7 +127,7 @@ enqMsg(uchar *msg, uchar *pszTag, int iFacility, int iSeverity, struct timeval * msgAddJSON(pMsg, (uchar*)"!", json); } - CHKiRet(submitMsg2(pMsg)); + CHKiRet(ratelimitAddMsg(ratelimiter, NULL, pMsg)); finalize_it: RETiRet; @@ -244,7 +250,14 @@ readjournal() { SD_JOURNAL_FOREACH_DATA(j, get, l) { /* locate equal sign, this is always present */ equal_sign = memchr(get, '=', l); - assert (equal_sign != NULL); + + /* ... but we know better than to trust the specs */ + if (equal_sign == NULL) { + errmsg.LogError(0, RS_RET_ERR,"SD_JOURNAL_FOREACH_DATA()" + " returned a malformed field (has no '='): '%s'", + (char*)get); + continue; /* skip the entry */ + } /* get length of journal data prefix */ prefixlen = ((char *)equal_sign - (char *)get); @@ -427,12 +440,13 @@ finalize_it: } -BEGINrunInput -CODESTARTrunInput - /* this is an endless loop - it is terminated when the thread is - * signalled to do so. This, however, is handled by the framework, - * right into the sleep below. - */ +/* This function loads a journal cursor from the state file. + */ +static rsRetVal +loadJournalState() +{ + DEFiRet; + if (cs.stateFile[0] != '/') { char *new_stateFile; @@ -472,6 +486,22 @@ CODESTARTrunInput } } +finalize_it: + RETiRet; +} + +BEGINrunInput +CODESTARTrunInput + CHKiRet(ratelimitNew(&ratelimiter, "imjournal", NULL)); + ratelimitSetLinuxLike(ratelimiter, cs.ratelimitInterval, cs.ratelimitBurst); + + if (cs.stateFile) { + CHKiRet(loadJournalState()); + } + + /* this is an endless loop - it is terminated when the thread is + * signalled to do so. This, however, is handled by the framework. + */ while (glbl.GetGlobalInputTermState() == 0) { int count = 0, r; @@ -492,11 +522,13 @@ CODESTARTrunInput } CHKiRet(readjournal()); - /* TODO: This could use some finer metric. */ - count++; - if (count == cs.iPersistStateInterval) { - count = 0; - persistJournalState(); + if (cs.stateFile) { /* can't persist without a state file */ + /* TODO: This could use some finer metric. */ + count++; + if (count == cs.iPersistStateInterval) { + count = 0; + persistJournalState(); + } } } @@ -510,6 +542,8 @@ CODESTARTbeginCnfLoad cs.iPersistStateInterval = DFLT_persiststateinterval; cs.stateFile = NULL; + cs.ratelimitBurst = 20000; + cs.ratelimitInterval = 600; ENDbeginCnfLoad @@ -545,13 +579,16 @@ ENDwillRun /* close journal */ BEGINafterRun CODESTARTafterRun - persistJournalState(); + if (cs.stateFile) { /* can't persist without a state file */ + persistJournalState(); + } sd_journal_close(j); ENDafterRun BEGINmodExit CODESTARTmodExit + ratelimitDestruct(ratelimiter); if(pInputName != NULL) prop.Destruct(&pInputName); if(pLocalHostIP != NULL) @@ -589,6 +626,10 @@ CODESTARTsetModCnf cs.iPersistStateInterval = (int) pvals[i].val.d.n; } else if (!strcmp(modpblk.descr[i].name, "statefile")) { cs.stateFile = (char *)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(modpblk.descr[i].name, "ratelimit.burst")) { + cs.ratelimitBurst = (int) pvals[i].val.d.n; + } else if(!strcmp(modpblk.descr[i].name, "ratelimit.interval")) { + cs.ratelimitInterval = (int) pvals[i].val.d.n; } else { dbgprintf("imjournal: program error, non-handled " "param '%s' in beginCnfLoad\n", modpblk.descr[i].name); @@ -634,6 +675,10 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"imjournalpersiststateinterval", 0, eCmdHdlrInt, NULL, &cs.iPersistStateInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imjournalratelimitinterval", 0, eCmdHdlrInt, + NULL, &cs.ratelimitInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imjournalratelimitburst", 0, eCmdHdlrInt, + NULL, &cs.ratelimitBurst, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imjournalstatefile", 0, eCmdHdlrGetWord, NULL, &cs.stateFile, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imzmq3/imzmq3.c b/plugins/imzmq3/imzmq3.c index 52ca5ebe..08b1dbe4 100644 --- a/plugins/imzmq3/imzmq3.c +++ b/plugins/imzmq3/imzmq3.c @@ -135,7 +135,6 @@ struct lstn_s { /* ---------------------------------------------------------------------------- * Static definitions/initializations. */ -static modConfData_t* loadModConf = NULL; static modConfData_t* runModConf = NULL; static struct lstn_s* lcnfRoot = NULL; static struct lstn_s* lcnfLast = NULL; @@ -268,7 +267,7 @@ static void setDefaults(instanceConf_t* info) { info->reconnectIVLMax = -1; info->ipv4Only = -1; info->affinity = -1; - + info->next = NULL; }; /* given a comma separated list of subscriptions, create a char* array of them @@ -442,11 +441,11 @@ static rsRetVal createInstance(instanceConf_t** pinst) { setDefaults(inst); /* add this to the config */ - if(loadModConf->tail == NULL) { - loadModConf->tail = loadModConf->root = inst; + if (runModConf->root == NULL || runModConf->tail == NULL) { + runModConf->tail = runModConf->root = inst; } else { - loadModConf->tail->next = inst; - loadModConf->tail = inst; + runModConf->tail->next = inst; + runModConf->tail = inst; } *pinst = inst; finalize_it: @@ -696,10 +695,14 @@ ENDisCompatibleWithFeature BEGINbeginCnfLoad CODESTARTbeginCnfLoad - loadModConf = pModConf; - pModConf->pConf = pConf; + /* After endCnfLoad() (BEGINendCnfLoad...ENDendCnfLoad) is called, + * the pModConf pointer must not be used to change the in-memory + * config object. It's safe to use the same pointer for accessing + * the config object until freeCnf() (BEGINfreeCnf...ENDfreeCnf). */ + runModConf = pModConf; + runModConf->pConf = pConf; /* init module config */ - loadModConf->io_threads = 0; /* 0 means don't set it */ + runModConf->io_threads = 0; /* 0 means don't set it */ ENDbeginCnfLoad @@ -718,7 +721,7 @@ CODESTARTsetModCnf if (!pvals[i].bUsed) continue; if (!strcmp(modpblk.descr[i].name, "ioThreads")) { - loadModConf->io_threads = (int)pvals[i].val.d.n; + runModConf->io_threads = (int)pvals[i].val.d.n; } else { errmsg.LogError(0, RS_RET_INVALID_PARAMS, "imzmq3: config error, unknown " @@ -735,7 +738,14 @@ ENDsetModCnf BEGINendCnfLoad CODESTARTendCnfLoad - loadModConf = NULL; /* done loading, so it becomes NULL */ + /* Last chance to make changes to the in-memory config object for this + * input module. After this call, the config object must no longer be + * changed. */ + if (pModConf != runModConf) { + errmsg.LogError(0, NO_ERRCODE, "imzmq3: pointer of in-memory config object has " + "changed - pModConf=%p, runModConf=%p", pModConf, runModConf); + } + assert(pModConf == runModConf); ENDendCnfLoad @@ -764,7 +774,12 @@ ENDcheckCnf BEGINactivateCnfPrePrivDrop CODESTARTactivateCnfPrePrivDrop - runModConf = pModConf; + if (pModConf != runModConf) { + errmsg.LogError(0, NO_ERRCODE, "imzmq3: pointer of in-memory config object has " + "changed - pModConf=%p, runModConf=%p", pModConf, runModConf); + } + assert(pModConf == runModConf); + /* first create the context */ createContext(); @@ -775,12 +790,41 @@ ENDactivateCnfPrePrivDrop BEGINactivateCnf CODESTARTactivateCnf + if (pModConf != runModConf) { + errmsg.LogError(0, NO_ERRCODE, "imzmq3: pointer of in-memory config object has " + "changed - pModConf=%p, runModConf=%p", pModConf, runModConf); + } + assert(pModConf == runModConf); ENDactivateCnf -/*TODO: Fill this in! */ BEGINfreeCnf + struct lstn_s *lstn, *lstn_r; + instanceConf_t *inst, *inst_r; + sublist *sub, *sub_r; CODESTARTfreeCnf + DBGPRINTF("imzmq3: BEGINfreeCnf ...\n"); + if (pModConf != runModConf) { + errmsg.LogError(0, NO_ERRCODE, "imzmq3: pointer of in-memory config object has " + "changed - pModConf=%p, runModConf=%p", pModConf, runModConf); + } + for (lstn = lcnfRoot; lstn != NULL; ) { + lstn_r = lstn; + lstn = lstn_r->next; + free(lstn_r); + } + for (inst = pModConf->root ; inst != NULL ; ) { + for (sub = inst->subscriptions; sub != NULL; ) { + free(sub->subscribe); + sub_r = sub; + sub = sub_r->next; + free(sub_r); + } + free(inst->pszBindRuleset); + inst_r = inst; + inst = inst->next; + free(inst_r); + } ENDfreeCnf diff --git a/rsyslog.service.in b/rsyslog.service.in index 08a4870f..8e2d64c2 100644 --- a/rsyslog.service.in +++ b/rsyslog.service.in @@ -1,9 +1,10 @@ [Unit] Description=System Logging Service +Requires=syslog.socket [Service] +Type=notify ExecStart=@sbindir@/rsyslogd -n -Sockets=syslog.socket StandardOutput=null [Install] diff --git a/runtime/ratelimit.c b/runtime/ratelimit.c index d83da2dd..6e1df3e2 100644 --- a/runtime/ratelimit.c +++ b/runtime/ratelimit.c @@ -128,8 +128,8 @@ tellLostCnt(ratelimit_t *ratelimit) snprintf((char*)msgbuf, sizeof(msgbuf), "%s: %u messages lost due to rate-limiting", ratelimit->name, ratelimit->missed); - logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); ratelimit->missed = 0; + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); } } @@ -157,9 +157,9 @@ withinRatelimit(ratelimit_t *ratelimit, time_t tt) /* resume if we go out of out time window */ if(tt > ratelimit->begin + ratelimit->interval) { - tellLostCnt(ratelimit); ratelimit->begin = 0; ratelimit->done = 0; + tellLostCnt(ratelimit); } /* do actual limit check */ @@ -167,13 +167,13 @@ withinRatelimit(ratelimit_t *ratelimit, time_t tt) ratelimit->done++; ret = 1; } else { - if(ratelimit->missed == 0) { + ratelimit->missed++; + if(ratelimit->missed == 1) { snprintf((char*)msgbuf, sizeof(msgbuf), "%s: begin to drop messages due to rate-limiting", ratelimit->name); logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); } - ratelimit->missed++; ret = 0; } diff --git a/tools/syslogd.c b/tools/syslogd.c index 7a8e21c2..a8a733d6 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2033,6 +2033,8 @@ int realMain(int argc, char **argv) ourConf->globals.bErrMsgToStderr = 0; } + sd_notify(0, "READY=1"); + mainloop(); /* do any de-init's that need to be done AFTER this comment */ |