From 03c2dbc7c25316c377dbf70bfff564070f0441d2 Mon Sep 17 00:00:00 2001 From: Milan Bartos Date: Thu, 15 Nov 2012 09:14:48 +0100 Subject: bugfix: imfile discarded some file parts File lines that were incomplete (LF missing) *at the time imfile polled the file* were partially discarded. That part of the line that was read without the LF was discarded, and the rest of the line was submitted in the next polling cycle. This is now changed so that the partial content is saved until the complete line is read. Note that the patch affects only read mode 0. --- ChangeLog | 8 ++++++++ runtime/stream.c | 27 ++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 077e2b6f..1c6d6e94 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,14 @@ Version 7.2.2 [v7-stable] 2012-10-?? aborted on startup. Note that no segfault could happen if the initial startup went well (this was a problem with the config parser). closes: http://bugzilla.adiscon.com/show_bug.cgi?id=381 +- bugfix: imfile discarded some file parts + File lines that were incomplete (LF missing) *at the time imfile polled + the file* were partially discarded. That part of the line that was read + without the LF was discarded, and the rest of the line was submitted in + the next polling cycle. This is now changed so that the partial content + is saved until the complete line is read. Note that the patch affects + only read mode 0. + Thanks to Milan Bartos for providing the patch. ---------------------------------------------------------------------------- Version 7.2.1 [v7-stable] 2012-10-29 - bugfix: ruleset()-object did only support a single statement diff --git a/runtime/stream.c b/runtime/stream.c index 906a45fc..52d143de 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -589,19 +589,40 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) uchar c; uchar finished; + rsRetVal readCharRet; + + static cstr_t *prevCStr = NULL; + ASSERT(pThis != NULL); ASSERT(ppCStr != NULL); CHKiRet(cstrConstruct(ppCStr)); - /* now read the line */ + /* append previous message to current message if necessary */ + if (prevCStr != NULL) { + CHKiRet(cstrAppendCStr(*ppCStr, prevCStr)); + } + CHKiRet(strmReadChar(pThis, &c)); if (mode == 0){ - while(c != '\n') { + while(c != '\n') { CHKiRet(cstrAppendChar(*ppCStr, c)); - CHKiRet(strmReadChar(pThis, &c)); + + readCharRet = strmReadChar(pThis, &c); + + /* end of file has been reached without \n */ + if (readCharRet == RS_RET_EOF) { + CHKiRet(rsCStrConstructFromCStr(&prevCStr, *ppCStr)); + } + + CHKiRet(readCharRet); } CHKiRet(cstrFinalize(*ppCStr)); + /* message has been finalized, destruct message from previous */ + if (prevCStr) { + cstrDestruct(&prevCStr); + prevCStr = NULL; + } } if (mode == 1){ finished=0; -- cgit v1.2.3 From 46dbd8dda97947713d76e814b3053760a106d63f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 15 Nov 2012 09:24:30 +0100 Subject: minor cleanup --- plugins/imfile/imfile.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 453b6b05..188d692b 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -308,14 +308,6 @@ finalize_it: /* submit everything that was not yet submitted */ CHKiRet(multiSubmitMsg(&pThis->multiSub)); } - ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ - /* Note: the problem above is that pthread:cleanup_pop() is a macro which - * evaluates to something like "} while(0);". So the code would become - * "finalize_it: }", that is a label without a statement. The C standard does - * not permit this. So we add an empty statement "finalize_it: ; }" and - * everybody is happy. Note that without the ;, an error is reported only - * on some platforms/compiler versions. -- rgerhards, 2008-08-15 - */ pthread_cleanup_pop(0); if(pCStr != NULL) { -- cgit v1.2.3 From 3475caa394c4d50baa34b407b7bdea0c37f2b83c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 15 Nov 2012 10:08:26 +0100 Subject: refactor stream.h; fix some issues with last patch Most importantly, the last patch for imfile contained a number of glitches, which are fixed by this commit (a memory leak under unusual conditions, partial message loss when rsyslog was terminated in the interim & mixing file data to the wrong files when multiple monitors were used [due to static variable]). The commit is actually a re-write of the patch, based on its core idea. Also some other minor cleanup was done. --- ChangeLog | 2 +- runtime/stream.c | 41 ++++++++++++++++------------------------- runtime/stream.h | 1 + 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1c6d6e94..a34ded48 100644 --- a/ChangeLog +++ b/ChangeLog @@ -48,7 +48,7 @@ Version 7.2.2 [v7-stable] 2012-10-?? the next polling cycle. This is now changed so that the partial content is saved until the complete line is read. Note that the patch affects only read mode 0. - Thanks to Milan Bartos for providing the patch. + Thanks to Milan Bartos for providing the base idea for the solution. ---------------------------------------------------------------------------- Version 7.2.1 [v7-stable] 2012-10-29 - bugfix: ruleset()-object did only support a single statement diff --git a/runtime/stream.c b/runtime/stream.c index 52d143de..193d14db 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -585,46 +585,33 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) * mode = 2 LF mode, a log line starts at the beginning of a line, but following lines that are indented are part of the same log entry * This modal interface is not nearly as flexible as being able to define a regex for when a new record starts, but it's also not nearly as hard (or as slow) to implement */ - DEFiRet; uchar c; uchar finished; - rsRetVal readCharRet; - - static cstr_t *prevCStr = NULL; + DEFiRet; ASSERT(pThis != NULL); ASSERT(ppCStr != NULL); CHKiRet(cstrConstruct(ppCStr)); + CHKiRet(strmReadChar(pThis, &c)); + if(mode == 0) { /* append previous message to current message if necessary */ - if (prevCStr != NULL) { - CHKiRet(cstrAppendCStr(*ppCStr, prevCStr)); + if(pThis->prevLineSegment != NULL) { + CHKiRet(cstrAppendCStr(*ppCStr, pThis->prevLineSegment)); + cstrDestruct(&pThis->prevLineSegment); } - - CHKiRet(strmReadChar(pThis, &c)); - if (mode == 0){ - while(c != '\n') { + while(c != '\n') { CHKiRet(cstrAppendChar(*ppCStr, c)); - readCharRet = strmReadChar(pThis, &c); - - /* end of file has been reached without \n */ - if (readCharRet == RS_RET_EOF) { - CHKiRet(rsCStrConstructFromCStr(&prevCStr, *ppCStr)); + if(readCharRet == RS_RET_EOF) {/* end of file reached without \n? */ + CHKiRet(rsCStrConstructFromCStr(&pThis->prevLineSegment, *ppCStr)); } - CHKiRet(readCharRet); } CHKiRet(cstrFinalize(*ppCStr)); - /* message has been finalized, destruct message from previous */ - if (prevCStr) { - cstrDestruct(&prevCStr); - prevCStr = NULL; - } - } - if (mode == 1){ + } else if(mode == 1) { finished=0; while(finished == 0){ if(c != '\n') { @@ -645,8 +632,7 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) } } CHKiRet(cstrFinalize(*ppCStr)); - } - if (mode == 2){ + } else if(mode == 2) { /* indented follow-up lines */ finished=0; while(finished == 0){ @@ -699,6 +685,7 @@ BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); pThis->tOpenMode = 0600; + pThis->prevLineSegment = NULL; ENDobjConstruct(strm) @@ -1606,6 +1593,8 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) l = pThis->iCurrOffs; objSerializeSCALAR_VAR(pStrm, iCurrOffs, INT64, l); + objSerializePTR(pStrm, prevLineSegment, PSZ); + CHKiRet(obj.EndSerialize(pStrm)); finalize_it: @@ -1711,6 +1700,8 @@ static rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) CHKiRet(strmSetiFileNumDigits(pThis, pProp->val.num)); } else if(isProp("bDeleteOnClose")) { CHKiRet(strmSetbDeleteOnClose(pThis, pProp->val.num)); + } else if(isProp("prevLineSegment")) { + CHKiRet(rsCStrConstructFromCStr(&pThis->prevLineSegment, pProp->val.pStr)); } finalize_it: diff --git a/runtime/stream.h b/runtime/stream.h index a01929f2..78dbc0d6 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -141,6 +141,7 @@ typedef struct strm_s { off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ sbool bIsTTY; /* is this a tty file? */ + cstr_t *prevLineSegment; /* for ReadLine, previous, unwritten part of file */ } strm_t; -- cgit v1.2.3 From 3cbb70651be422f44f34664637e1a88aab732dd5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 16 Nov 2012 14:50:26 +0100 Subject: prepare 7.2.2 release --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- tests/diag.sh | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index a34ded48..20585f10 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ ---------------------------------------------------------------------------- -Version 7.2.2 [v7-stable] 2012-10-?? +Version 7.2.2 [v7-stable] 2012-10-16 - doc improvements - enabled to build without libuuid, at loss of uuid functionality this enables smoother builds on older systems that do not support diff --git a/configure.ac b/configure.ac index 91571719..e55da7bd 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[7.2.1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[7.2.2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 185e873e..58570097 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

-

This documentation is for version 7.2.1 (v7-stable branch) of rsyslog. +

This documentation is for version 7.2.2 (v7-stable branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

If you like rsyslog, you might diff --git a/tests/diag.sh b/tests/diag.sh index 02b24c5b..bd38b29d 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -10,8 +10,8 @@ #valgrind="valgrind --tool=helgrind --log-fd=1" #valgrind="valgrind --tool=exp-ptrcheck --log-fd=1" #set -o xtrace -export RSYSLOG_DEBUG="debug nologfuncflow noprintmutexaction nostdout" -export RSYSLOG_DEBUGLOG="log" +#export RSYSLOG_DEBUG="debug nologfuncflow noprintmutexaction nostdout" +#export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason cp $srcdir/testsuites/diag-common.conf diag-common.conf -- cgit v1.2.3 From 6898e319118bed0e6e6ef3881947a7859fc17912 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 20 Nov 2012 10:02:34 +0100 Subject: cleanup --- grammar/lexer.l | 1 - 1 file changed, 1 deletion(-) diff --git a/grammar/lexer.l b/grammar/lexer.l index ce7c34c5..63eaa8ce 100644 --- a/grammar/lexer.l +++ b/grammar/lexer.l @@ -88,7 +88,6 @@ extern int yydebug; /* somehow, I need these prototype even though the headers are * included. I guess that's some autotools magic I don't understand... */ -//char *strdup(char*); int fileno(FILE *stream); %} -- cgit v1.2.3 From 1d3c4454403eb038220f8fd075f0bc836b2e03ed Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 20 Nov 2012 10:08:06 +0100 Subject: regression fix: rsyslog terminated when wild-card $includeFile did not find files Unfortunately, this is often the case by default in many distros. --- grammar/rainerscript.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 27ff5376..1116c913 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -2757,6 +2757,9 @@ cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) return func; } +/* returns 0 if everything is OK and config parsing shall continue, + * and 1 if things are so wrong that config parsing shall be aborted. + */ int cnfDoInclude(char *name) { @@ -2784,7 +2787,7 @@ cnfDoInclude(char *name) /* Silently ignore wildcards that match nothing */ if(result == GLOB_NOMATCH) { - return 1; + return 0; } if(result == GLOB_NOSPACE || result == GLOB_ABORTED) { @@ -2802,7 +2805,7 @@ cnfDoInclude(char *name) rs_strerror_r(errno, errStr, sizeof(errStr)); parser_errmsg("error accessing config file or directory '%s': %s", cfgFile, errStr); - continue; + return 1; } if(S_ISREG(fileInfo.st_mode)) { /* config file */ -- cgit v1.2.3 From 9c25e344f2734c7f82fdad43c2fd88e91b27c315 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 20 Nov 2012 10:15:36 +0100 Subject: document fixed regression --- ChangeLog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ChangeLog b/ChangeLog index 20585f10..ca37a0b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,16 @@ ---------------------------------------------------------------------------- +Version 7.2.3 [v7-stable] 2012-10-?? +- regression fix: rsyslogd terminated when wild-card $IncludeConfig did not + find actual include files. For example, if this directive is present: + $IncludeConfig /etc/rsyslog.d/*.conf + and there are no *.conf files in /etc/rsyslog.d (but rsyslog.d exists), + rsyslogd will emit an error message and terminate. Previous (and expected) + behaviour is that an empty file set is no problem. HOWEVER, if the + directory itself does not exist, this is flagged as an error and will + load to termination (no startup). + Unfortunately, this is often the case by default in many distros, so this + actually prevents rsyslog startup. +---------------------------------------------------------------------------- Version 7.2.2 [v7-stable] 2012-10-16 - doc improvements - enabled to build without libuuid, at loss of uuid functionality -- cgit v1.2.3 From 4a857f4629d5ca4ff4c59875bc798e70b1811ba9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 20 Nov 2012 14:31:35 +0100 Subject: improve $IncludeConfig error messages --- grammar/rainerscript.c | 11 +++++++---- tests/incltest_dir_empty_wildcard.sh | 13 +++++++++++++ tests/testsuites/incltest_dir_empty_wildcard.conf | 11 +++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100755 tests/incltest_dir_empty_wildcard.sh create mode 100644 tests/testsuites/incltest_dir_empty_wildcard.conf diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 1116c913..0dc505a7 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -2770,6 +2770,7 @@ cnfDoInclude(char *name) glob_t cfgFiles; struct stat fileInfo; char nameBuf[MAXFNAME+1]; + char cwdBuf[MAXFNAME+1]; finalName = name; if(stat(name, &fileInfo) == 0) { @@ -2793,8 +2794,9 @@ cnfDoInclude(char *name) if(result == GLOB_NOSPACE || result == GLOB_ABORTED) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - parser_errmsg("error accessing config file or directory '%s': %s", - finalName, errStr); + getcwd(cwdBuf, sizeof(cwdBuf)); + parser_errmsg("error accessing config file or directory '%s' [cwd:%s]: %s", + finalName, cwdBuf, errStr); return 1; } @@ -2803,8 +2805,9 @@ cnfDoInclude(char *name) if(stat(cfgFile, &fileInfo) != 0) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - parser_errmsg("error accessing config file or directory '%s': %s", - cfgFile, errStr); + getcwd(cwdBuf, sizeof(cwdBuf)); + parser_errmsg("error accessing config file or directory '%s' " + "[cwd: %s]: %s", cfgFile, cwdBuf, errStr); return 1; } diff --git a/tests/incltest_dir_empty_wildcard.sh b/tests/incltest_dir_empty_wildcard.sh new file mode 100755 index 00000000..6cdb3b21 --- /dev/null +++ b/tests/incltest_dir_empty_wildcard.sh @@ -0,0 +1,13 @@ +# This test checks if an empty includeConfig directory causes problems. It +# should not, as this is a valid situation that by default exists on many +# distros. +echo =============================================================================== +echo \[incltest_dir_empty_wildcard.sh\]: test $IncludeConfig for \"empty\" wildcard +source $srcdir/diag.sh init +source $srcdir/diag.sh startup incltest_dir_empty_wildcard.conf +# 100 messages are enough - the question is if the include is read ;) +source $srcdir/diag.sh injectmsg 0 100 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/incltest_dir_empty_wildcard.conf b/tests/testsuites/incltest_dir_empty_wildcard.conf new file mode 100644 index 00000000..5e750c5b --- /dev/null +++ b/tests/testsuites/incltest_dir_empty_wildcard.conf @@ -0,0 +1,11 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# the following include leads to no files actually being included +# but MUST NOT abort rsyslog's startup sequence. No files matching +# the wildcard is valid (as long as the path exists)! +$IncludeConfig testsuites/incltest.d/*.conf-not-there + +$template outfmt,"%msg:F,58:2%\n" +:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt -- cgit v1.2.3 From 51c4130853ca0da2cc3729e959b66915c8a82391 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 20 Nov 2012 14:32:00 +0100 Subject: testbench: bug fix & enhance $IncludeConfig tests The bug fixes actually fixes a bug inside the *testbench* itself. --- tests/Makefile.am | 3 +++ tests/diag.sh | 5 ++++- tests/imptcp_addtlframedelim.sh | 1 - tests/imptcp_conndrop.sh | 3 +-- tests/imptcp_large.sh | 1 - tests/imtcp_addtlframedelim.sh | 1 - tests/testsuites/imptcp_addtlframedelim.conf | 1 - tests/testsuites/imptcp_conndrop.conf | 1 - tests/testsuites/imptcp_large.conf | 1 - tests/testsuites/imtcp_addtlframedelim.conf | 1 - 10 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index a673ef1e..e3226c6d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -76,6 +76,7 @@ TESTS += \ incltest.sh \ incltest_dir.sh \ incltest_dir_wildcard.sh \ + incltest_dir_empty_wildcard.sh \ linkedlistqueue.sh if HAVE_VALGRIND @@ -298,6 +299,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/incltest.conf \ incltest_dir.sh \ testsuites/incltest_dir.conf \ + incltest_dir_empty_wildcard.sh \ + testsuites/incltest_dir_empty_wildcard.conf \ incltest_dir_wildcard.sh \ testsuites/incltest_dir_wildcard.conf \ testsuites/incltest.d/include.conf \ diff --git a/tests/diag.sh b/tests/diag.sh index bd38b29d..2fdcbfb9 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -16,7 +16,6 @@ case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason cp $srcdir/testsuites/diag-common.conf diag-common.conf cp $srcdir/testsuites/diag-common2.conf diag-common2.conf - rm -f rsyslog.action.*.include rm -f rsyslogd.started work-*.conf rsyslog.random.data rm -f rsyslogd2.started work-*.conf rm -f work rsyslog.out.log rsyslog2.out.log rsyslog.out.log.save # common work files @@ -24,6 +23,10 @@ case $1 in rm -f rsyslog.out.*.log work-presort rsyslog.pipe rm -f rsyslog.input rsyslog.empty rm -f core.* vgcore.* + # Note: rsyslog.action.*.include must NOT be deleted, as it + # is used to setup some parameters BEFORE calling init. This + # happens in chained test scripts. Delete on exit is fine, + # though. mkdir test-spool ;; 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf diff --git a/tests/imptcp_addtlframedelim.sh b/tests/imptcp_addtlframedelim.sh index b26fc85b..00276ab3 100755 --- a/tests/imptcp_addtlframedelim.sh +++ b/tests/imptcp_addtlframedelim.sh @@ -3,7 +3,6 @@ # This file is part of the rsyslog project, released under GPLv3 echo ==================================================================================== echo TEST: \[imptcp_addtlframedelim.sh\]: test imptcp additional frame delimiter -cat rsyslog.action.1.include source $srcdir/diag.sh init source $srcdir/diag.sh startup imptcp_addtlframedelim.conf source $srcdir/diag.sh tcpflood -m20000 -F0 -P129 diff --git a/tests/imptcp_conndrop.sh b/tests/imptcp_conndrop.sh index 684de6b5..0cf0ba5e 100755 --- a/tests/imptcp_conndrop.sh +++ b/tests/imptcp_conndrop.sh @@ -4,9 +4,8 @@ # This file is part of the rsyslog project, released under GPLv3 echo ==================================================================================== echo TEST: \[imptcp_conndrop.sh\]: test imptcp with random connection drops -cat rsyslog.action.1.include source $srcdir/diag.sh init -source $srcdir/diag.sh startup imptcp_large.conf +source $srcdir/diag.sh startup imptcp_conndrop.conf # 100 byte messages to gain more practical data use source $srcdir/diag.sh tcpflood -c20 -m50000 -r -d100 -P129 -D sleep 4 # due to large messages, we need this time for the tcp receiver to settle... diff --git a/tests/imptcp_large.sh b/tests/imptcp_large.sh index b4d130bb..43027069 100755 --- a/tests/imptcp_large.sh +++ b/tests/imptcp_large.sh @@ -4,7 +4,6 @@ # This file is part of the rsyslog project, released under GPLv3 echo ==================================================================================== echo TEST: \[imptcp_large.sh\]: test imptcp with large-size messages -cat rsyslog.action.1.include source $srcdir/diag.sh init source $srcdir/diag.sh startup imptcp_large.conf # send 4000 messages of 10.000bytes plus header max, randomized diff --git a/tests/imtcp_addtlframedelim.sh b/tests/imtcp_addtlframedelim.sh index 8de7ca58..4c1fd9cb 100755 --- a/tests/imtcp_addtlframedelim.sh +++ b/tests/imtcp_addtlframedelim.sh @@ -3,7 +3,6 @@ # This file is part of the rsyslog project, released under GPLv3 echo ==================================================================================== echo TEST: \[imtcp_addtlframedelim.sh\]: test imtcp additional frame delimiter -cat rsyslog.action.1.include source $srcdir/diag.sh init source $srcdir/diag.sh startup imtcp_addtlframedelim.conf source $srcdir/diag.sh tcpflood -m20000 -F0 -P129 diff --git a/tests/testsuites/imptcp_addtlframedelim.conf b/tests/testsuites/imptcp_addtlframedelim.conf index eb7ed0c4..bf302fb4 100644 --- a/tests/testsuites/imptcp_addtlframedelim.conf +++ b/tests/testsuites/imptcp_addtlframedelim.conf @@ -9,5 +9,4 @@ $template outfmt,"%msg:F,58:2%\n" $OMFileFlushOnTXEnd off $OMFileFlushInterval 2 $OMFileIOBufferSize 256k -$IncludeConfig rsyslog.action.1.include local0.* ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/imptcp_conndrop.conf b/tests/testsuites/imptcp_conndrop.conf index 677e33f6..77a5d79a 100644 --- a/tests/testsuites/imptcp_conndrop.conf +++ b/tests/testsuites/imptcp_conndrop.conf @@ -12,5 +12,4 @@ $template dynfile,"rsyslog.out.log" # trick to use relative path names! $OMFileFlushOnTXEnd off $OMFileFlushInterval 2 $OMFileIOBufferSize 256k -$IncludeConfig rsyslog.action.1.include local0.* ?dynfile;outfmt diff --git a/tests/testsuites/imptcp_large.conf b/tests/testsuites/imptcp_large.conf index 677e33f6..77a5d79a 100644 --- a/tests/testsuites/imptcp_large.conf +++ b/tests/testsuites/imptcp_large.conf @@ -12,5 +12,4 @@ $template dynfile,"rsyslog.out.log" # trick to use relative path names! $OMFileFlushOnTXEnd off $OMFileFlushInterval 2 $OMFileIOBufferSize 256k -$IncludeConfig rsyslog.action.1.include local0.* ?dynfile;outfmt diff --git a/tests/testsuites/imtcp_addtlframedelim.conf b/tests/testsuites/imtcp_addtlframedelim.conf index 3b4759c5..6558c519 100644 --- a/tests/testsuites/imtcp_addtlframedelim.conf +++ b/tests/testsuites/imtcp_addtlframedelim.conf @@ -9,5 +9,4 @@ $template outfmt,"%msg:F,58:2%\n" $OMFileFlushOnTXEnd off $OMFileFlushInterval 2 $OMFileIOBufferSize 256k -$IncludeConfig rsyslog.action.1.include local0.* ./rsyslog.out.log;outfmt -- cgit v1.2.3 From 2ed0f99ce2576c4b52cccd7cdc0d17531281f85f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 21 Nov 2012 17:49:55 +0100 Subject: prepare for 7.2.3 release --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca37a0b4..a9e7e179 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ ---------------------------------------------------------------------------- -Version 7.2.3 [v7-stable] 2012-10-?? +Version 7.2.3 [v7-stable] 2012-10-21 - regression fix: rsyslogd terminated when wild-card $IncludeConfig did not find actual include files. For example, if this directive is present: $IncludeConfig /etc/rsyslog.d/*.conf diff --git a/configure.ac b/configure.ac index e55da7bd..4e6827e2 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[7.2.2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[7.2.3],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 58570097..f15557d3 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

-

This documentation is for version 7.2.2 (v7-stable branch) of rsyslog. +

This documentation is for version 7.2.3 (v7-stable branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

If you like rsyslog, you might -- cgit v1.2.3 From 318a6fb577a6e5af558b70232bb0a19871399d13 Mon Sep 17 00:00:00 2001 From: Marius Tomaschewski Date: Wed, 21 Nov 2012 13:46:58 +0100 Subject: imklog: added $klogKeepKernelTimestamp option When enabled, the kernel [timestamp] remains at begin of each message, even it is used for the message time too. --- doc/imklog.html | 5 +++++ plugins/imklog/bsd.c | 6 ++++-- plugins/imklog/imklog.c | 3 +++ plugins/imklog/imklog.h | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/imklog.html b/doc/imklog.html index 2e3b3bc2..268c661a 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -65,6 +65,11 @@ Linux only, ignored on other platforms (but may be specified) former klogd -2 option
Linux only, ignored on other platforms (but may be specified)
+

  • $klogKeepKernelTimestamp [on/off] +If enabled, this option causes to keep the [timestamp] provided by the kernel at the begin +of in each message rather than to remove it, when it could be parsed and converted into +local time for use as regular message time. +
  • Caveats/Known Bugs:

    This is obviously platform specific and requires platform diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index bb45c97a..ec4110da 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -119,8 +119,10 @@ submitSyslog(int pri, uchar *buf) /* we have a timestamp */ DBGPRINTF("kernel timestamp is %ld %ld\n", secs, nsecs); - bufsize= strlen((char*)buf); - memmove(buf+3, buf+i, bufsize - i + 1); + if(!bKeepKernelStamp) { + bufsize= strlen((char*)buf); + memmove(buf+3, buf+i, bufsize - i + 1); + } clock_gettime(CLOCK_MONOTONIC, &monotonic); clock_gettime(CLOCK_REALTIME, &realtime); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index aa500084..563b4994 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -77,6 +77,7 @@ int symbols_twice = 0; int use_syscall = 0; int symbol_lookup = 0; /* on recent kernels > 2.6, the kernel does this */ int bPermitNonKernel = 0; /* permit logging of messages not having LOG_KERN facility */ +int bKeepKernelStamp = 0; /* keep the kernel timestamp in the message */ int iFacilIntMsg; /* the facility to use for internal messages (set by driver) */ uchar *pszPath = NULL; int console_log_level = -1; @@ -305,6 +306,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a symfile = NULL; symbol_lookup = 0; bPermitNonKernel = 0; + bKeepKernelStamp = 0; if(pszPath != NULL) { free(pszPath); pszPath = NULL; @@ -332,6 +334,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogpermitnonkernelfacility", 0, eCmdHdlrBinary, NULL, &bPermitNonKernel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogconsoleloglevel", 0, eCmdHdlrInt, NULL, &console_log_level, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"kloginternalmsgfacility", 0, eCmdHdlrFacility, NULL, &iFacilIntMsg, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogkeepkerneltimestamp", 0, eCmdHdlrBinary, NULL, &bKeepKernelStamp, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit /* vim:set ai: diff --git a/plugins/imklog/imklog.h b/plugins/imklog/imklog.h index 7a060df4..03fb9735 100644 --- a/plugins/imklog/imklog.h +++ b/plugins/imklog/imklog.h @@ -52,6 +52,7 @@ extern char *symfile; extern int console_log_level; extern int dbgPrintSymbols; extern uchar *pszPath; +extern int bKeepKernelStamp; /* the functions below may be called by the drivers */ rsRetVal imklogLogIntMsg(int priority, char *fmt, ...) __attribute__((format(printf,2, 3))); -- cgit v1.2.3 From f040bde7a0454dfbc7def36a288bc70c58d878af Mon Sep 17 00:00:00 2001 From: Marius Tomaschewski Date: Wed, 21 Nov 2012 13:47:19 +0100 Subject: imklog: added $klogParseKernelTimestamp option When enabled, kernel message [timestamp] is converted for message time. Default is to use receive time as in 5.8.x and before, because the clock used to create the timestamp is not supposed to be as accurate as the monotonic clock (depends on hardware and kernel) resulting in differences between kernel and system messages which occurred at same time. --- doc/imklog.html | 10 +++++++++- plugins/imklog/bsd.c | 3 +++ plugins/imklog/imklog.c | 3 +++ plugins/imklog/imklog.h | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/imklog.html b/doc/imklog.html index 268c661a..05292ddf 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -65,10 +65,18 @@ Linux only, ignored on other platforms (but may be specified) former klogd -2 option
    Linux only, ignored on other platforms (but may be specified)
    +

  • $klogParseKernelTimestamp [on/off] +If enabled and the kernel creates a timestamp for its log messages, this timestamp will be +parsed and converted into regular message time instead to use the receive time of the kernel +message (as in 5.8.x and before). Default is to not parse the kernel timestamp, because the +clock used by the kernel to create the timestamps is not supposed to be as accurate as the +monotonic clock required to convert it. Depending on the hardware and kernel, it can result +in message time differences between kernel and system messages which occurred at same time. +
  • $klogKeepKernelTimestamp [on/off] If enabled, this option causes to keep the [timestamp] provided by the kernel at the begin of in each message rather than to remove it, when it could be parsed and converted into -local time for use as regular message time. +local time for use as regular message time. Only used, when $klogParseKernelTimestamp is on.
  • Caveats/Known Bugs: diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index ec4110da..06032373 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -85,6 +85,9 @@ submitSyslog(int pri, uchar *buf) struct timeval tv; struct timeval *tp = NULL; + if(!bParseKernelStamp) + goto done; + if(buf[3] != '[') goto done; DBGPRINTF("imklog: kernel timestamp detected, extracting it\n"); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 563b4994..e9536519 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -77,6 +77,7 @@ int symbols_twice = 0; int use_syscall = 0; int symbol_lookup = 0; /* on recent kernels > 2.6, the kernel does this */ int bPermitNonKernel = 0; /* permit logging of messages not having LOG_KERN facility */ +int bParseKernelStamp = 0; /* if try to parse kernel timestamps for message time */ int bKeepKernelStamp = 0; /* keep the kernel timestamp in the message */ int iFacilIntMsg; /* the facility to use for internal messages (set by driver) */ uchar *pszPath = NULL; @@ -306,6 +307,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a symfile = NULL; symbol_lookup = 0; bPermitNonKernel = 0; + bParseKernelStamp = 0; bKeepKernelStamp = 0; if(pszPath != NULL) { free(pszPath); @@ -334,6 +336,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogpermitnonkernelfacility", 0, eCmdHdlrBinary, NULL, &bPermitNonKernel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogconsoleloglevel", 0, eCmdHdlrInt, NULL, &console_log_level, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"kloginternalmsgfacility", 0, eCmdHdlrFacility, NULL, &iFacilIntMsg, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogparsekerneltimestamp", 0, eCmdHdlrBinary, NULL, &bParseKernelStamp, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogkeepkerneltimestamp", 0, eCmdHdlrBinary, NULL, &bKeepKernelStamp, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/imklog/imklog.h b/plugins/imklog/imklog.h index 03fb9735..c93a53b5 100644 --- a/plugins/imklog/imklog.h +++ b/plugins/imklog/imklog.h @@ -52,6 +52,7 @@ extern char *symfile; extern int console_log_level; extern int dbgPrintSymbols; extern uchar *pszPath; +extern int bParseKernelStamp; extern int bKeepKernelStamp; /* the functions below may be called by the drivers */ -- cgit v1.2.3 From b78af2aaf546cee90671513be39d8e3717f2ace6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 22 Nov 2012 08:57:57 +0100 Subject: bugfix: imklog mistakenly took kernel timestamp subseconds as nanoseconds ... actually, they are microseconds. So the fractional part of the timestamp was not properly formatted. Thanks to Marius Tomaschwesky for the bug report and the patch idea. --- ChangeLog | 15 +++++++++++++++ plugins/imklog/bsd.c | 24 ++++++++++++------------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index f4bec22e..55f75264 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,21 @@ Version 5.10.2 [V5-STABLE], 201?-??-?? - bugfix: spurios error messages from imuxsock about (non-error) EAGAIN Thanks to Marius Tomaschwesky for the patch. +- imklog: added $klogParseKernelTimestamp option + When enabled, kernel message [timestamp] is converted for message time. + Default is to use receive time as in 5.8.x and before, because the clock + used to create the timestamp is not supposed to be as accurate as the + monotonic clock (depends on hardware and kernel) resulting in differences + between kernel and system messages which occurred at same time. + Thanks to Marius Tomaschwesky for the patch. +- imklog: added $klogKeepKernelTimestamp option + When enabled, the kernel [timestamp] remains at begin of + each message, even it is used for the message time too. + Thanks to Marius Tomaschwesky for the patch. +- bugfix: imklog mistakenly took kernel timestamp subseconds as nanoseconds + ... actually, they are microseconds. So the fractional part of the + timestamp was not properly formatted. + Thanks to Marius Tomaschwesky for the bug report and the patch idea. --------------------------------------------------------------------------- Version 5.10.1 [V5-STABLE], 2012-10-17 - bugfix: imuxsock and imklog truncated head of received message diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index 06032373..17d4bead 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -76,9 +76,9 @@ static void submitSyslog(int pri, uchar *buf) { long secs; - long nsecs; + long usecs; long secOffs; - long nsecOffs; + long usecOffs; unsigned i; unsigned bufsize; struct timespec monotonic, realtime; @@ -109,9 +109,9 @@ submitSyslog(int pri, uchar *buf) } ++i; /* skip dot */ - nsecs = 0; + usecs = 0; while(buf[i] && isdigit(buf[i])) { - nsecs = nsecs * 10 + buf[i] - '0'; + usecs = usecs * 10 + buf[i] - '0'; ++i; } if(buf[i] != ']') { @@ -121,7 +121,7 @@ submitSyslog(int pri, uchar *buf) ++i; /* skip ']' */ /* we have a timestamp */ - DBGPRINTF("kernel timestamp is %ld %ld\n", secs, nsecs); + DBGPRINTF("kernel timestamp is %ld %ld\n", secs, usecs); if(!bKeepKernelStamp) { bufsize= strlen((char*)buf); memmove(buf+3, buf+i, bufsize - i + 1); @@ -130,20 +130,20 @@ submitSyslog(int pri, uchar *buf) clock_gettime(CLOCK_MONOTONIC, &monotonic); clock_gettime(CLOCK_REALTIME, &realtime); secOffs = realtime.tv_sec - monotonic.tv_sec; - nsecOffs = realtime.tv_nsec - monotonic.tv_nsec; - if(nsecOffs < 0) { + usecOffs = (realtime.tv_nsec - monotonic.tv_nsec) / 1000; + if(usecOffs < 0) { secOffs--; - nsecOffs += 1000000000l; + usecOffs += 1000000l; } - nsecs +=nsecOffs; - if(nsecs > 999999999l) { + usecs += usecOffs; + if(usecs > 999999l) { secs++; - nsecs -= 1000000000l; + usecs -= 1000000l; } secs += secOffs; tv.tv_sec = secs; - tv.tv_usec = nsecs / 1000; + tv.tv_usec = usecs; tp = &tv; done: -- cgit v1.2.3 From ae7b78cdb27d0477c59252808d535f0beb4c541c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 22 Nov 2012 10:08:43 +0100 Subject: imklog: add new config params to module() parameter set This is based on Marius Tomaschewski's set of patches, mostly just applied manually (as I need to mangle with the merge). --- plugins/imklog/imklog.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 7a1078f8..a24fc63b 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -94,6 +94,8 @@ static struct cnfparamdescr modpdescr[] = { { "logpath", eCmdHdlrGetWord, 0 }, { "permitnonkernelfacility", eCmdHdlrBinary, 0 }, { "consoleloglevel", eCmdHdlrInt, 0 }, + { "parsekerneltimestamp", eCmdHdlrBinary, 0 }, + { "keepkerneltimestamp", eCmdHdlrBinary, 0 }, { "internalmsgfacility", eCmdHdlrFacility, 0 } }; static struct cnfparamblk modpblk = @@ -323,6 +325,10 @@ CODESTARTsetModCnf loadModConf->pszPath = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(modpblk.descr[i].name, "permitnonkernelfacility")) { loadModConf->bPermitNonKernel = (int) pvals[i].val.d.n; + } else if(!strcmp(modpblk.descr[i].name, "parsekerneltimestamp")) { + loadModConf->bParseKernelStamp = (int) pvals[i].val.d.n; + } else if(!strcmp(modpblk.descr[i].name, "keepkerneltimestamp")) { + loadModConf->bKeepKernelStamp = (int) pvals[i].val.d.n; } else if(!strcmp(modpblk.descr[i].name, "consoleloglevel")) { loadModConf->console_log_level= (int) pvals[i].val.d.n; } else if(!strcmp(modpblk.descr[i].name, "internalmsgfacility")) { -- cgit v1.2.3 From 59c7a299e9907f81f0970d3523a47f4361466d08 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 22 Nov 2012 10:46:10 +0100 Subject: doc: mention imported patches in ChangeLog --- ChangeLog | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 86291232..f9feb70e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,14 @@ ---------------------------------------------------------------------------- +Version 7.2.4 [v7-stable] 2012-10-?? +- imklog: added ParseKernelTimestamp parameter (import from 5.10.2) + Thanks to Marius Tomaschweski for the patch. +- imklog: added KeepKernelTimestamp parameter (import from 5.10.2) + Thanks to Marius Tomaschweski for the patch. +- bugfix: imklog mistakenly took kernel timestamp subseconds as nanoseconds + ... actually, they are microseconds. So the fractional part of the + timestamp was not properly formatted. (import from 5.10.2) + Thanks to Marius Tomaschweski for the bug report and the patch idea. +---------------------------------------------------------------------------- Version 7.2.3 [v7-stable] 2012-10-21 - regression fix: rsyslogd terminated when wild-card $IncludeConfig did not find actual include files. For example, if this directive is present: @@ -90,12 +100,12 @@ Version 7.1.12 [beta] 2012-10-18 Thanks to Michael Biebl for reporting & suggestions - bugfix: imklog truncated head of received message This happened only under some circumstances. Thanks to Marius - Tomaschwesky and Florian Piekert for their help in solving this issue. + Tomaschewski and Florian Piekert for their help in solving this issue. ---------------------------------------------------------------------------- Version 7.1.11 [beta] 2012-10-16 - bugfix: imuxsock truncated head of received message This happened only under some circumstances. Thanks to Marius - Tomaschwesky, Florian Piekert and Milan Bartos for their help in + Tomaschewski, Florian Piekert and Milan Bartos for their help in solving this issue. - bugfix: do not crash if set statement is used with date field Thanks to Miloslav Trmač for the patch. -- cgit v1.2.3