From aef0aba9cd00fb225d2803210586b86a13547ce0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Aug 2010 12:49:59 +0200 Subject: fix required for new engine --- plugins/imptcp/imptcp.c | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 732590a9..905067ea 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -494,7 +494,6 @@ doSubmitMsg(ptcpsess_t *pThis, struct syslogTime *stTime, time_t ttGenTime, mult MsgSetInputName(pMsg, pThis->pSrv->pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pThis->peerName); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->peerIP)); MsgSetRuleset(pMsg, pThis->pSrv->pRuleset); -- cgit v1.2.3 From 41327992680cfb8dd602ea92d6902448dd66413d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 7 Sep 2010 13:06:04 +0200 Subject: acquire /dev/log socket optionally from systemd --- ChangeLog | 2 ++ plugins/imuxsock/Makefile.am | 4 +-- plugins/imuxsock/imuxsock.c | 50 ++++++++++++++++++++++++++++++++---- runtime/Makefile.am | 5 ++++ tools/Makefile.am | 2 +- tools/syslogd.c | 61 +++++++++++++++++++++++--------------------- 6 files changed, 87 insertions(+), 37 deletions(-) diff --git a/ChangeLog b/ChangeLog index da9379c2..f4bdc6ae 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- acquire /dev/log socket optionally from systemd + thanks to Lennart Poettering for this patch --------------------------------------------------------------------------- Version 5.5.6 [DEVEL] (rgerhards), 2010-06-?? - added parser modules diff --git a/plugins/imuxsock/Makefile.am b/plugins/imuxsock/Makefile.am index a2fe0baa..8803937f 100644 --- a/plugins/imuxsock/Makefile.am +++ b/plugins/imuxsock/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imuxsock.la -imuxsock_la_SOURCES = imuxsock.c +imuxsock_la_SOURCES = imuxsock.c ../../runtime/sd-daemon.c ../../runtime/sd-daemon.h imuxsock_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imuxsock_la_LDFLAGS = -module -avoid-version -imuxsock_la_LIBADD = +imuxsock_la_LIBADD = diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 046f12f0..db53fcb6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -47,6 +47,7 @@ #include "prop.h" #include "debug.h" #include "unlimited_select.h" +#include "sd-daemon.h" MODULE_TYPE_INPUT @@ -74,7 +75,7 @@ DEFobjCurrIf(prop) static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -static int startIndexUxLocalSockets; /* process funix from that index on (used to +static int startIndexUxLocalSockets; /* process funix from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup */ @@ -177,7 +178,7 @@ static rsRetVal discardFunixn(void) prop.Destruct(&(funixHName[i])); } } - + return RS_RET_OK; } @@ -190,6 +191,40 @@ static int create_unix_socket(const char *path, int bCreatePath) if (path[0] == '\0') return -1; + if (strcmp(path, _PATH_LOG) == 0) { + int r; + + /* Check whether an FD was passed in from systemd. If + * so, it's the /dev/log socket, so use it. */ + + r = sd_listen_fds(0); + if (r < 0) { + errmsg.LogError(-r, NO_ERRCODE, "Failed to acquire systemd socket"); + return -1; + } + + if (r > 1) { + errmsg.LogError(EINVAL, NO_ERRCODE, "Wrong number of systemd sockets passed"); + return -1; + } + + if (r == 1) { + fd = SD_LISTEN_FDS_START; + r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); + if (r < 0) { + errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); + return -1; + } + + if (!r) { + errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); + return -1; + } + + return fd; + } + } + unlink(path); memset(&sunx, 0, sizeof(sunx)); @@ -391,12 +426,17 @@ CODESTARTafterRun int i; /* do cleanup here */ /* Close the UNIX sockets. */ - for (i = 0; i < nfunix; i++) + for (i = 0; i < nfunix; i++) if (funix[i] != -1) close(funix[i]); - /* Clean-up files. */ - for(i = startIndexUxLocalSockets; i < nfunix; i++) + /* Clean-up files. If systemd passed us a socket it is + * systemd's job to clean it up.*/ + if (sd_listen_fds(0) > 0) + i = 1; + else + i = startIndexUxLocalSockets; + for(; i < nfunix; i++) if (funixn[i] && funix[i] != -1) unlink((char*) funixn[i]); /* free no longer needed string */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index f7db3e35..ef119492 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -81,6 +81,8 @@ librsyslog_la_SOURCES = \ prop.h \ cfsysline.c \ cfsysline.h \ + sd-daemon.c \ + sd-daemon.h \ \ \ ../action.h \ @@ -177,3 +179,6 @@ lmnsd_gtls_la_LDFLAGS = -module -avoid-version lmnsd_gtls_la_LIBADD = $(GNUTLS_LIBS) endif +update-systemd: + curl http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c > sd-daemon.c + curl http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h > sd-daemon.h diff --git a/tools/Makefile.am b/tools/Makefile.am index 8f2989ca..96657ad4 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -1,5 +1,5 @@ sbin_PROGRAMS = -man_MANS = rsyslogd.8 rsyslog.conf.5 +man_MANS = rsyslogd.8 rsyslog.conf.5 sbin_PROGRAMS += rsyslogd rsyslogd_SOURCES = \ diff --git a/tools/syslogd.c b/tools/syslogd.c index 9b7b77ab..a7f76313 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -135,6 +135,7 @@ #include "net.h" #include "vm.h" #include "prop.h" +#include "sd-daemon.h" /* definitions for objects we access */ DEFobjCurrIf(obj) @@ -391,7 +392,7 @@ static char **crunch_list(char *list) char **result = NULL; p = list; - + /* strip off trailing delimiters */ while (p[strlen(p)-1] == LIST_DELIMITER) { count--; @@ -400,18 +401,18 @@ static char **crunch_list(char *list) /* cut off leading delimiters */ while (p[0] == LIST_DELIMITER) { count--; - p++; + p++; } - + /* count delimiters to calculate elements */ for (count=i=0; p[i]; i++) if (p[i] == LIST_DELIMITER) count++; - + if ((result = (char **)MALLOC(sizeof(char *) * (count+2))) == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ } - + /* * We now can assume that the first and last * characters are different from any delimiters, @@ -574,7 +575,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) if(bHaveMainQueue == 0) { /* not yet in queued mode */ iminternalAddMsg(pri, pMsg); } else { - /* we have the queue, so we can simply provide the + /* we have the queue, so we can simply provide the * message to the queue engine. */ submitMsg(pMsg); @@ -721,7 +722,7 @@ submitMsg(msg_t *pMsg) DEFiRet; ISOBJ_TYPE_assert(pMsg, msg); - + pRuleset = MsgGetRuleset(pMsg); pQueue = (pRuleset == NULL) ? pMsgQueue : ruleset.GetRulesetQueue(pRuleset); @@ -787,7 +788,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) { action_t *pAction = (action_t*) pData; assert(pAction != NULL); - + BEGINfunc LockObj(pAction); /* TODO: time() performance: the call below could be moved to @@ -842,7 +843,7 @@ static void debug_switch() dbgprintf("\n"); debugging_on = 0; } - + memset(&sigAct, 0, sizeof (sigAct)); sigemptyset(&sigAct.sa_mask); sigAct.sa_handler = debug_switch; @@ -1030,7 +1031,7 @@ destructAllActions(void) /* die() is called when the program shall end. This typically only occurs - * during sigterm or during the initialization. + * during sigterm or during the initialization. * As die() is intended to shutdown rsyslogd, it is * safe to call exit() here. Just make sure that die() itself is not called * at inapropriate places. As a general rule of thumb, it is a bad idea to add @@ -1070,7 +1071,7 @@ die(int sig) errno = 0; logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)buf, 0); } - + /* drain queue (if configured so) and stop main queue worker thread pool */ DBGPRINTF("Terminating main queue...\n"); qqueueDestruct(&pMsgQueue); @@ -1194,7 +1195,7 @@ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) } -/* drop to specified group +/* drop to specified group * if something goes wrong, the function never returns * Note that such an abort can cause damage to on-disk structures, so we should * re-design the "interface" in the long term. -- rgerhards, 2008-11-26 @@ -1222,7 +1223,7 @@ static void doDropPrivGid(int iGid) } -/* drop to specified user +/* drop to specified user * if something goes wrong, the function never returns * Note that such an abort can cause damage to on-disk structures, so we should * re-design the "interface" in the long term. -- rgerhards, 2008-11-19 @@ -1332,7 +1333,7 @@ generateConfigDAG(uchar *pszDAGFile) DEFiRet; assert(pszDAGFile != NULL); - + if((fp = fopen((char*) pszDAGFile, "w")) == NULL) { logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*) "configuraton graph output file could not be opened, none generated", 0); @@ -1458,7 +1459,7 @@ static void dbgPrintInitInfo(void) /* TODO: add iActionRetryCount = 0; iActionRetryInterval = 30000; - static int iMainMsgQtoWrkMinMsgs = 100; + static int iMainMsgQtoWrkMinMsgs = 100; static int iMainMsgQbSaveOnShutdown = 1; iMainMsgQueMaxDiskSpace = 0; setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); @@ -1730,7 +1731,7 @@ init(void) * identify this instance. -- rgerhards, 2005-08-17 */ if(bLogStatusMsgs) { - snprintf(bufStartUpMsg, sizeof(bufStartUpMsg)/sizeof(char), + snprintf(bufStartUpMsg, sizeof(bufStartUpMsg)/sizeof(char), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] start", (int) myPid); @@ -1760,7 +1761,7 @@ finalize_it: -/* Put the rsyslog main thread to sleep for n seconds. This was introduced as +/* Put the rsyslog main thread to sleep for n seconds. This was introduced as * a quick and dirty workaround for a privilege drop race in regard to listener * startup, which itself was a result of the not-yet-done proper coding of * privilege drop code (quite some effort). It may be useful for other occasions, too. @@ -1845,7 +1846,7 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz void sighup_handler() { struct sigaction sigAct; - + bHadHUP = 1; memset(&sigAct, 0, sizeof (sigAct)); @@ -2428,8 +2429,10 @@ doGlblProcessInit(void) num_fds = getdtablesize(); close(0); /* we keep stdout and stderr open in case we have to emit something */ - for (i = 3; i < num_fds; i++) - (void) close(i); + + if (sd_listen_fds(0) <= 0) + for (i = 3; i < num_fds; i++) + (void) close(i); untty(); } else @@ -2486,7 +2489,7 @@ doGlblProcessInit(void) * modularize it a bit more... */ int realMain(int argc, char **argv) -{ +{ DEFiRet; register uchar *p; @@ -2516,7 +2519,7 @@ int realMain(int argc, char **argv) * of other options, we do this during the inital option processing. With later * versions (if a dependency on -c option is introduced), we must move that code * to other places, but I think it is quite appropriate and saves code to do this - * only when actually neeeded. + * only when actually neeeded. * rgerhards, 2008-04-04 */ while((ch = getopt(argc, argv, "46a:Ac:def:g:hi:l:m:M:nN:op:qQr::s:t:T:u:vwx")) != EOF) { @@ -2581,7 +2584,7 @@ int realMain(int argc, char **argv) case 'v': /* MUST be carried out immediately! */ printVersion(); exit(0); /* exit for -v option - so this is a "good one" */ - case '?': + case '?': default: usage(); } @@ -2637,7 +2640,7 @@ int realMain(int argc, char **argv) * Good software also always checks its return values... * If syslogd starts up before DNS is up & /etc/hosts * doesn't have LocalHostName listed, gethostbyname will - * return NULL. + * return NULL. */ /* TODO: gethostbyname() is not thread-safe, but replacing it is * not urgent as we do not run on multiple threads here. rgerhards, 2007-09-25 @@ -2646,7 +2649,7 @@ int realMain(int argc, char **argv) if(hent) { free(LocalHostName); CHKmalloc(LocalHostName = (uchar*)strdup(hent->h_name)); - + if((p = (uchar*)strchr((char*)LocalHostName, '.'))) { *p++ = '\0'; @@ -2658,7 +2661,7 @@ int realMain(int argc, char **argv) /* Convert to lower case to recognize the correct domain laterly */ for(p = LocalDomain ; *p ; p++) *p = (char)tolower((int)*p); - + /* we now have our hostname and can set it inside the global vars. * TODO: think if all of this would better be a runtime function * rgerhards, 2008-04-17 @@ -2814,7 +2817,7 @@ int realMain(int argc, char **argv) case 'x': /* disable dns for remote messages */ glbl.SetDisableDNS(1); break; - case '?': + case '?': default: usage(); } @@ -2871,7 +2874,7 @@ int realMain(int argc, char **argv) /* do any de-init's that need to be done AFTER this comment */ die(bFinished); - + thrdExit(); finalize_it: @@ -2893,7 +2896,7 @@ finalize_it: * rgerhards, 20080-01-28 */ int main(int argc, char **argv) -{ +{ dbgClassInit(); return realMain(argc, argv); } -- cgit v1.2.3 From 3e2d011e8c1c42801194d387b75ca4f4a175ae5d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Sep 2010 14:19:05 +0200 Subject: added forgotten files --- runtime/sd-daemon.c | 448 ++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/sd-daemon.h | 257 ++++++++++++++++++++++++++++++ 2 files changed, 705 insertions(+) create mode 100644 runtime/sd-daemon.c create mode 100644 runtime/sd-daemon.h diff --git a/runtime/sd-daemon.c b/runtime/sd-daemon.c new file mode 100644 index 00000000..5df70e38 --- /dev/null +++ b/runtime/sd-daemon.c @@ -0,0 +1,448 @@ +/*-*- Mode: C; c-basic-offset: 8 -*-*/ + +/*** + Copyright 2010 Lennart Poettering + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +***/ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sd-daemon.h" + +int sd_listen_fds(int unset_environment) { + +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + int r, fd; + const char *e; + char *p = NULL; + unsigned long l; + + if (!(e = getenv("LISTEN_PID"))) { + r = 0; + goto finish; + } + + errno = 0; + l = strtoul(e, &p, 10); + + if (errno != 0) { + r = -errno; + goto finish; + } + + if (!p || *p || l <= 0) { + r = -EINVAL; + goto finish; + } + + /* Is this for us? */ + if (getpid() != (pid_t) l) { + r = 0; + goto finish; + } + + if (!(e = getenv("LISTEN_FDS"))) { + r = 0; + goto finish; + } + + errno = 0; + l = strtoul(e, &p, 10); + + if (errno != 0) { + r = -errno; + goto finish; + } + + if (!p || *p) { + r = -EINVAL; + goto finish; + } + + for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + (int) l; fd ++) { + int flags; + + if ((flags = fcntl(fd, F_GETFD)) < 0) { + r = -errno; + goto finish; + } + + if (flags & FD_CLOEXEC) + continue; + + if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) { + r = -errno; + goto finish; + } + } + + r = (int) l; + +finish: + if (unset_environment) { + unsetenv("LISTEN_PID"); + unsetenv("LISTEN_FDS"); + } + + return r; +#endif +} + +int sd_is_fifo(int fd, const char *path) { + struct stat st_fd; + + if (fd < 0) + return -EINVAL; + + memset(&st_fd, 0, sizeof(st_fd)); + if (fstat(fd, &st_fd) < 0) + return -errno; + + if (!S_ISFIFO(st_fd.st_mode)) + return 0; + + if (path) { + struct stat st_path; + + memset(&st_path, 0, sizeof(st_path)); + if (stat(path, &st_path) < 0) { + + if (errno == ENOENT || errno == ENOTDIR) + return 0; + + return -errno; + } + + return + st_path.st_dev == st_fd.st_dev && + st_path.st_ino == st_fd.st_ino; + } + + return 1; +} + +static int sd_is_socket_internal(int fd, int type, int listening) { + struct stat st_fd; + + if (fd < 0 || type < 0) + return -EINVAL; + + if (fstat(fd, &st_fd) < 0) + return -errno; + + if (!S_ISSOCK(st_fd.st_mode)) + return 0; + + if (type != 0) { + int other_type = 0; + socklen_t l = sizeof(other_type); + + if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &other_type, &l) < 0) + return -errno; + + if (l != sizeof(other_type)) + return -EINVAL; + + if (other_type != type) + return 0; + } + + if (listening >= 0) { + int accepting = 0; + socklen_t l = sizeof(accepting); + + if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &accepting, &l) < 0) + return -errno; + + if (l != sizeof(accepting)) + return -EINVAL; + + if (!accepting != !listening) + return 0; + } + + return 1; +} + +union sockaddr_union { + struct sockaddr sa; + struct sockaddr_in in4; + struct sockaddr_in6 in6; + struct sockaddr_un un; + struct sockaddr_storage storage; +}; + +int sd_is_socket(int fd, int family, int type, int listening) { + int r; + + if (family < 0) + return -EINVAL; + + if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) + return r; + + if (family > 0) { + union sockaddr_union sockaddr; + socklen_t l; + + memset(&sockaddr, 0, sizeof(sockaddr)); + l = sizeof(sockaddr); + + if (getsockname(fd, &sockaddr.sa, &l) < 0) + return -errno; + + if (l < sizeof(sa_family_t)) + return -EINVAL; + + return sockaddr.sa.sa_family == family; + } + + return 1; +} + +int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) { + union sockaddr_union sockaddr; + socklen_t l; + int r; + + if (family != 0 && family != AF_INET && family != AF_INET6) + return -EINVAL; + + if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) + return r; + + memset(&sockaddr, 0, sizeof(sockaddr)); + l = sizeof(sockaddr); + + if (getsockname(fd, &sockaddr.sa, &l) < 0) + return -errno; + + if (l < sizeof(sa_family_t)) + return -EINVAL; + + if (sockaddr.sa.sa_family != AF_INET && + sockaddr.sa.sa_family != AF_INET6) + return 0; + + if (family > 0) + if (sockaddr.sa.sa_family != family) + return 0; + + if (port > 0) { + if (sockaddr.sa.sa_family == AF_INET) { + if (l < sizeof(struct sockaddr_in)) + return -EINVAL; + + return htons(port) == sockaddr.in4.sin_port; + } else { + if (l < sizeof(struct sockaddr_in6)) + return -EINVAL; + + return htons(port) == sockaddr.in6.sin6_port; + } + } + + return 1; +} + +int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) { + union sockaddr_union sockaddr; + socklen_t l; + int r; + + if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) + return r; + + memset(&sockaddr, 0, sizeof(sockaddr)); + l = sizeof(sockaddr); + + if (getsockname(fd, &sockaddr.sa, &l) < 0) + return -errno; + + if (l < sizeof(sa_family_t)) + return -EINVAL; + + if (sockaddr.sa.sa_family != AF_UNIX) + return 0; + + if (path) { + if (length <= 0) + length = strlen(path); + + if (length <= 0) + /* Unnamed socket */ + return l == sizeof(sa_family_t); + + if (path[0]) + /* Normal path socket */ + return + (l >= sizeof(sa_family_t) + length + 1) && + memcmp(path, sockaddr.un.sun_path, length+1) == 0; + else + /* Abstract namespace socket */ + return + (l == sizeof(sa_family_t) + length) && + memcmp(path, sockaddr.un.sun_path, length) == 0; + } + + return 1; +} + +int sd_notify(int unset_environment, const char *state) { +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + int fd = -1, r; + struct msghdr msghdr; + struct iovec iovec; + union sockaddr_union sockaddr; + struct ucred *ucred; + union { + struct cmsghdr cmsghdr; + uint8_t buf[CMSG_SPACE(sizeof(struct ucred))]; + } control; + const char *e; + + if (!state) { + r = -EINVAL; + goto finish; + } + + if (!(e = getenv("NOTIFY_SOCKET"))) + return 0; + + /* Must be an abstract socket, or an absolute path */ + if ((e[0] != '@' && e[0] != '/') || e[1] == 0) { + r = -EINVAL; + goto finish; + } + + if ((fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) { + r = -errno; + goto finish; + } + + memset(&sockaddr, 0, sizeof(sockaddr)); + sockaddr.sa.sa_family = AF_UNIX; + strncpy(sockaddr.un.sun_path, e, sizeof(sockaddr.un.sun_path)); + + if (sockaddr.un.sun_path[0] == '@') + sockaddr.un.sun_path[0] = 0; + + memset(&iovec, 0, sizeof(iovec)); + iovec.iov_base = (char*) state; + iovec.iov_len = strlen(state); + + memset(&control, 0, sizeof(control)); + control.cmsghdr.cmsg_level = SOL_SOCKET; + control.cmsghdr.cmsg_type = SCM_CREDENTIALS; + control.cmsghdr.cmsg_len = CMSG_LEN(sizeof(struct ucred)); + + ucred = (struct ucred*) CMSG_DATA(&control.cmsghdr); + ucred->pid = getpid(); + ucred->uid = getuid(); + ucred->gid = getgid(); + + memset(&msghdr, 0, sizeof(msghdr)); + msghdr.msg_name = &sockaddr; + msghdr.msg_namelen = sizeof(struct sockaddr_un); + msghdr.msg_iov = &iovec; + msghdr.msg_iovlen = 1; + msghdr.msg_control = &control; + msghdr.msg_controllen = control.cmsghdr.cmsg_len; + + if (sendmsg(fd, &msghdr, MSG_NOSIGNAL) < 0) { + r = -errno; + goto finish; + } + + r = 1; + +finish: + if (unset_environment) + unsetenv("NOTIFY_SOCKET"); + + if (fd >= 0) + close(fd); + + return r; +#endif +} + +int sd_notifyf(int unset_environment, const char *format, ...) { +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + va_list ap; + char *p = NULL; + int r; + + va_start(ap, format); + r = vasprintf(&p, format, ap); + va_end(ap); + + if (r < 0 || !p) + return -ENOMEM; + + r = sd_notify(unset_environment, p); + free(p); + + return r; +#endif +} + +int sd_booted(void) { +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + + struct stat a, b; + + /* We simply test whether the systemd cgroup hierarchy is + * mounted */ + + if (lstat("/cgroup", &a) < 0) + return 0; + + if (lstat("/cgroup/systemd", &b) < 0) + return 0; + + return a.st_dev != b.st_dev; +#endif +} diff --git a/runtime/sd-daemon.h b/runtime/sd-daemon.h new file mode 100644 index 00000000..fd6221f2 --- /dev/null +++ b/runtime/sd-daemon.h @@ -0,0 +1,257 @@ +/*-*- Mode: C; c-basic-offset: 8 -*-*/ + +#ifndef foosddaemonhfoo +#define foosddaemonhfoo + +/*** + Copyright 2010 Lennart Poettering + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +***/ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + Reference implementation of a few systemd related interfaces for + writing daemons. These interfaces are trivial to implement. To + simplify porting we provide this reference implementation. + Applications are welcome to reimplement the algorithms described + here if they do not want to include these two source files. + + The following functionality is provided: + + - Support for logging with log levels on stderr + - File descriptor passing for socket-based activation + - Daemon startup and status notification + - Detection of systemd boots + + You may compile this with -DDISABLE_SYSTEMD to disable systemd + support. This makes all those calls NOPs that are directly related to + systemd (i.e. only sd_is_xxx() will stay useful). + + Since this is drop-in code we don't want any of our symbols to be + exported in any case. Hence we declare hidden visibility for all of + them. + + You may find an up-to-date version of these source files online: + + http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h + http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c + + This should compile on non-Linux systems, too, but with the + exception of the sd_is_xxx() calls all functions will become NOPs. + + See sd-daemon(7) for more information. +*/ + +#if __GNUC__ >= 4 +#define _sd_printf_attr_(a,b) __attribute__ ((format (printf, a, b))) +#define _sd_hidden_ __attribute__ ((visibility("hidden"))) +#else +#define _sd_printf_attr_(a,b) +#define _sd_hidden_ +#endif + +/* + Log levels for usage on stderr: + + fprintf(stderr, SD_NOTICE "Hello World!\n"); + + This is similar to printk() usage in the kernel. +*/ +#define SD_EMERG "<0>" /* system is unusable */ +#define SD_ALERT "<1>" /* action must be taken immediately */ +#define SD_CRIT "<2>" /* critical conditions */ +#define SD_ERR "<3>" /* error conditions */ +#define SD_WARNING "<4>" /* warning conditions */ +#define SD_NOTICE "<5>" /* normal but significant condition */ +#define SD_INFO "<6>" /* informational */ +#define SD_DEBUG "<7>" /* debug-level messages */ + +/* The first passed file descriptor is fd 3 */ +#define SD_LISTEN_FDS_START 3 + +/* + Returns how many file descriptors have been passed, or a negative + errno code on failure. Optionally, removes the $LISTEN_FDS and + $LISTEN_PID file descriptors from the environment (recommended, but + problematic in threaded environments). If r is the return value of + this function you'll find the file descriptors passed as fds + SD_LISTEN_FDS_START to SD_LISTEN_FDS_START+r-1. Returns a negative + errno style error code on failure. This function call ensures that + the FD_CLOEXEC flag is set for the passed file descriptors, to make + sure they are not passed on to child processes. If FD_CLOEXEC shall + not be set, the caller needs to unset it after this call for all file + descriptors that are used. + + See sd_listen_fds(3) for more information. +*/ +int sd_listen_fds(int unset_environment) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is a FIFO in the file system stored under the + specified path, 0 otherwise. If path is NULL a path name check will + not be done and the call only verifies if the file descriptor + refers to a FIFO. Returns a negative errno style error code on + failure. + + See sd_is_fifo(3) for more information. +*/ +int sd_is_fifo(int fd, const char *path) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is a socket of the specified family (AF_INET, + ...) and type (SOCK_DGRAM, SOCK_STREAM, ...), 0 otherwise. If + family is 0 a socket family check will not be done. If type is 0 a + socket type check will not be done and the call only verifies if + the file descriptor refers to a socket. If listening is > 0 it is + verified that the socket is in listening mode. (i.e. listen() has + been called) If listening is == 0 it is verified that the socket is + not in listening mode. If listening is < 0 no listening mode check + is done. Returns a negative errno style error code on failure. + + See sd_is_socket(3) for more information. +*/ +int sd_is_socket(int fd, int family, int type, int listening) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is an Internet socket, of the specified family + (either AF_INET or AF_INET6) and the specified type (SOCK_DGRAM, + SOCK_STREAM, ...), 0 otherwise. If version is 0 a protocol version + check is not done. If type is 0 a socket type check will not be + done. If port is 0 a socket port check will not be done. The + listening flag is used the same way as in sd_is_socket(). Returns a + negative errno style error code on failure. + + See sd_is_socket_inet(3) for more information. +*/ +int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is an AF_UNIX socket of the specified type + (SOCK_DGRAM, SOCK_STREAM, ...) and path, 0 otherwise. If type is 0 + a socket type check will not be done. If path is NULL a socket path + check will not be done. For normal AF_UNIX sockets set length to + 0. For abstract namespace sockets set length to the length of the + socket name (including the initial 0 byte), and pass the full + socket path in path (including the initial 0 byte). The listening + flag is used the same way as in sd_is_socket(). Returns a negative + errno style error code on failure. + + See sd_is_socket_unix(3) for more information. +*/ +int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) _sd_hidden_; + +/* + Informs systemd about changed daemon state. This takes a number of + newline seperated environment-style variable assignments in a + string. The following variables are known: + + READY=1 Tells systemd that daemon startup is finished (only + relevant for services of Type=notify). The passed + argument is a boolean "1" or "0". Since there is + little value in signalling non-readiness the only + value daemons should send is "READY=1". + + STATUS=... Passes a single-line status string back to systemd + that describes the daemon state. This is free-from + and can be used for various purposes: general state + feedback, fsck-like programs could pass completion + percentages and failing programs could pass a human + readable error message. Example: "STATUS=Completed + 66% of file system check..." + + ERRNO=... If a daemon fails, the errno-style error code, + formatted as string. Example: "ERRNO=2" for ENOENT. + + BUSERROR=... If a daemon fails, the D-Bus error-style error + code. Example: "BUSERROR=org.freedesktop.DBus.Error.TimedOut" + + MAINPID=... The main pid of a daemon, in case systemd did not + fork off the process itself. Example: "MAINPID=4711" + + Daemons can choose to send additional variables. However, it is + recommened to prefix variable names not listed above with X_. + + Returns a negative errno-style error code on failure. Returns > 0 + if systemd could be notified, 0 if it couldn't possibly because + systemd is not running. + + Example: When a daemon finished starting up, it could issue this + call to notify systemd about it: + + sd_notify(0, "READY=1"); + + See sd_notifyf() for more complete examples. + + See sd_notify(3) for more information. +*/ +int sd_notify(int unset_environment, const char *state) _sd_hidden_; + +/* + Similar to sd_notify() but takes a format string. + + Example 1: A daemon could send the following after initialization: + + sd_notifyf(0, "READY=1\n" + "STATUS=Processing requests...\n" + "MAINPID=%lu", + (unsigned long) getpid()); + + Example 2: A daemon could send the following shortly before + exiting, on failure: + + sd_notifyf(0, "STATUS=Failed to start up: %s\n" + "ERRNO=%i", + strerror(errno), + errno); + + See sd_notifyf(3) for more information. +*/ +int sd_notifyf(int unset_environment, const char *format, ...) _sd_printf_attr_(2,3) _sd_hidden_; + +/* + Returns > 0 if the system was booted with systemd. Returns < 0 on + error. Returns 0 if the system was not booted with systemd. Note + that all of the functions above handle non-systemd boots just + fine. You should NOT protect them with a call to this function. Also + note that this function checks whether the system, not the user + session is controlled by systemd. However the functions above work + for both session and system services. + + See sd_booted(3) for more information. +*/ +int sd_booted(void) _sd_hidden_; + +#ifdef __cplusplus +} +#endif + +#endif -- cgit v1.2.3 From 21a2672222a1b403ff0811154dec156676bed732 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Sep 2010 12:46:03 +0200 Subject: moved systemd interface to rsyslog convenience lib Mostly a refresh of sd-daemon.[ch] from its source plus some make file changes. We now have systemd interfaces inside rsyslog, so that all plugins interested can call the interfaces. Seems not to be totally necessary right now, but will help in the long term. --- plugins/imuxsock/Makefile.am | 6 +++--- runtime/Makefile.am | 4 ++-- runtime/sd-daemon.c | 31 +++++++++---------------------- runtime/sd-daemon.h | 12 ++++++++---- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/plugins/imuxsock/Makefile.am b/plugins/imuxsock/Makefile.am index 8803937f..28f9f9e3 100644 --- a/plugins/imuxsock/Makefile.am +++ b/plugins/imuxsock/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imuxsock.la -imuxsock_la_SOURCES = imuxsock.c ../../runtime/sd-daemon.c ../../runtime/sd-daemon.h -imuxsock_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +imuxsock_la_SOURCES = imuxsock.c +imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imuxsock_la_LDFLAGS = -module -avoid-version -imuxsock_la_LIBADD = +imuxsock_la_LIBADD = $(RSRT_LIBS) diff --git a/runtime/Makefile.am b/runtime/Makefile.am index ef119492..5b2598b2 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -101,9 +101,9 @@ librsyslog_la_SOURCES = \ # runtime or will no longer be needed. -- rgerhards, 2008-06-13 if WITH_MODDIRS -librsyslog_la_CPPFLAGS = -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS) +librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS) else -librsyslog_la_CPPFLAGS = -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS) +librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS) endif #librsyslog_la_LDFLAGS = -module -avoid-version librsyslog_la_LIBADD = $(DL_LIBS) $(RT_LIBS) diff --git a/runtime/sd-daemon.c b/runtime/sd-daemon.c index 5df70e38..9c23b917 100644 --- a/runtime/sd-daemon.c +++ b/runtime/sd-daemon.c @@ -1,4 +1,4 @@ -/*-*- Mode: C; c-basic-offset: 8 -*-*/ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ /*** Copyright 2010 Lennart Poettering @@ -325,18 +325,13 @@ int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t } int sd_notify(int unset_environment, const char *state) { -#if defined(DISABLE_SYSTEMD) || !defined(__linux__) +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) || !defined(SOCK_CLOEXEC) return 0; #else int fd = -1, r; struct msghdr msghdr; struct iovec iovec; union sockaddr_union sockaddr; - struct ucred *ucred; - union { - struct cmsghdr cmsghdr; - uint8_t buf[CMSG_SPACE(sizeof(struct ucred))]; - } control; const char *e; if (!state) { @@ -369,23 +364,15 @@ int sd_notify(int unset_environment, const char *state) { iovec.iov_base = (char*) state; iovec.iov_len = strlen(state); - memset(&control, 0, sizeof(control)); - control.cmsghdr.cmsg_level = SOL_SOCKET; - control.cmsghdr.cmsg_type = SCM_CREDENTIALS; - control.cmsghdr.cmsg_len = CMSG_LEN(sizeof(struct ucred)); - - ucred = (struct ucred*) CMSG_DATA(&control.cmsghdr); - ucred->pid = getpid(); - ucred->uid = getuid(); - ucred->gid = getgid(); - memset(&msghdr, 0, sizeof(msghdr)); msghdr.msg_name = &sockaddr; - msghdr.msg_namelen = sizeof(struct sockaddr_un); + msghdr.msg_namelen = sizeof(sa_family_t) + strlen(e); + + if (msghdr.msg_namelen > sizeof(struct sockaddr_un)) + msghdr.msg_namelen = sizeof(struct sockaddr_un); + msghdr.msg_iov = &iovec; msghdr.msg_iovlen = 1; - msghdr.msg_control = &control; - msghdr.msg_controllen = control.cmsghdr.cmsg_len; if (sendmsg(fd, &msghdr, MSG_NOSIGNAL) < 0) { r = -errno; @@ -437,10 +424,10 @@ int sd_booted(void) { /* We simply test whether the systemd cgroup hierarchy is * mounted */ - if (lstat("/cgroup", &a) < 0) + if (lstat("/sys/fs/cgroup", &a) < 0) return 0; - if (lstat("/cgroup/systemd", &b) < 0) + if (lstat("/sys/fs/cgroup/systemd", &b) < 0) return 0; return a.st_dev != b.st_dev; diff --git a/runtime/sd-daemon.h b/runtime/sd-daemon.h index fd6221f2..45aac8bd 100644 --- a/runtime/sd-daemon.h +++ b/runtime/sd-daemon.h @@ -1,4 +1,4 @@ -/*-*- Mode: C; c-basic-offset: 8 -*-*/ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ #ifndef foosddaemonhfoo #define foosddaemonhfoo @@ -67,9 +67,13 @@ extern "C" { See sd-daemon(7) for more information. */ -#if __GNUC__ >= 4 +#if (__GNUC__ >= 4) #define _sd_printf_attr_(a,b) __attribute__ ((format (printf, a, b))) -#define _sd_hidden_ __attribute__ ((visibility("hidden"))) +# if defined(SD_EXPORT_SYMBOLS) +# define _sd_hidden_ +# else +# define _sd_hidden_ __attribute__ ((visibility("hidden"))) +# endif #else #define _sd_printf_attr_(a,b) #define _sd_hidden_ @@ -171,7 +175,7 @@ int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t /* Informs systemd about changed daemon state. This takes a number of - newline seperated environment-style variable assignments in a + newline separated environment-style variable assignments in a string. The following variables are known: READY=1 Tells systemd that daemon startup is finished (only -- cgit v1.2.3 From 947e5632039d1dd2ef3ae22ac45b17370bbac24d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Sep 2010 13:26:33 +0200 Subject: bumped version number (new v5-devel) --- ChangeLog | 3 +++ configure.ac | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 1b6d57cc..cfd1831d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +--------------------------------------------------------------------------- +Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-?? +- support for systemd officially added - acquire /dev/log socket optionally from systemd thanks to Lennart Poettering for this patch --------------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index 9e0aad5c..9c005a19 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.5.7],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) -- cgit v1.2.3 From 3cbd7300c47991aadca00db1b028fd3d5d551040 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Sep 2010 12:24:22 +0200 Subject: added module impstat to emit periodic statistics on rsyslog counters This is a *very first* and *very rough* and *very featureless* first shot at this functionality. It is assumed that we will enhance the stats system as a by-line while doing other development. --- ChangeLog | 6 +- Makefile.am | 5 + configure.ac | 15 +++ doc/Makefile.am | 1 + doc/impstats.html | 60 ++++++++++++ doc/rsyslog_conf_modules.html | 1 + plugins/impstats/Makefile.am | 6 ++ plugins/impstats/impstats.c | 211 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 doc/impstats.html create mode 100644 plugins/impstats/Makefile.am create mode 100644 plugins/impstats/impstats.c diff --git a/ChangeLog b/ChangeLog index cfd1831d..0f8b625d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,10 @@ --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-?? +- added module impstat to emit periodic statistics on rsyslog counters - support for systemd officially added -- acquire /dev/log socket optionally from systemd - thanks to Lennart Poettering for this patch + * acquire /dev/log socket optionally from systemd + thanks to Lennart Poettering for this patch + * sd-systemd API added as part of rsyslog runtime library --------------------------------------------------------------------------- Version 5.5.7 [V5-BETA] (rgerhards), 2010-08-09 - changed omudpspoof default spoof address to simplify typical use case diff --git a/Makefile.am b/Makefile.am index 76a38322..bc6d8dd0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -59,6 +59,10 @@ if ENABLE_IMKLOG SUBDIRS += plugins/imklog endif +if ENABLE_IMPSTATS +SUBDIRS += plugins/impstats +endif + if ENABLE_IMSOLARIS SUBDIRS += plugins/imsolaris endif @@ -191,5 +195,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-omuxsock \ --enable-shave \ --enable-extended-tests \ + --enable-impstats \ --enable-memcheck ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 9c005a19..c52749c8 100644 --- a/configure.ac +++ b/configure.ac @@ -792,6 +792,19 @@ AC_ARG_ENABLE(imptcp, AM_CONDITIONAL(ENABLE_IMPTCP, test x$enable_imptcp = xyes) +# settings for the pstats input module +AC_ARG_ENABLE(impstats, + [AS_HELP_STRING([--enable-impstats],[periodic statistics module enabled @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_impstats="yes" ;; + no) enable_impstats="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-impstats) ;; + esac], + [enable_impstats=no] +) +AM_CONDITIONAL(ENABLE_IMPSTATS, test x$enable_impstats = xyes) + + # settings for the omprog output module AC_ARG_ENABLE(omprog, [AS_HELP_STRING([--enable-omprog],[Compiles omprog module @<:@default=no@:>@])], @@ -1013,6 +1026,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imfile/Makefile \ plugins/imsolaris/Makefile \ plugins/imptcp/Makefile \ + plugins/impstats/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ plugins/omtesting/Makefile \ @@ -1049,6 +1063,7 @@ echo " plain tcp input module enabled: $enable_imptcp" echo " imdiag enabled: $enable_imdiag" echo " file input module enabled: $enable_imfile" echo " Solaris input module enabled: $enable_imsolaris" +echo " periodic statistics module enabled: $enable_impstats" echo " input template module will be compiled: $enable_imtemplate" echo echo "---{ output plugins }---" diff --git a/doc/Makefile.am b/doc/Makefile.am index d4df740a..a5393cbe 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -41,6 +41,7 @@ html_files = \ imfile.html \ imtcp.html \ imptcp.html \ + impstats.html \ imgssapi.html \ imrelp.html \ imsolaris.html \ diff --git a/doc/impstats.html b/doc/impstats.html new file mode 100644 index 00000000..0f4096ab --- /dev/null +++ b/doc/impstats.html @@ -0,0 +1,60 @@ + + +Periodic Statistics of Internal Counters (impstats) + + +back + +

Input Module to Generate Periodic Statistics of Internal Counters

+

Module Name:    impstats

+

Available since: 5.7.0+, 6.1.1+ +

Author: Rainer Gerhards <rgerhards@adiscon.com>

+

Description:

+

This module provides periodic output of rsyslog internal counters. +Note that the whole statistics system is currently under development. So +availabilty and format of counters may change and is not yet stable (so be +prepared to change your trending scripts when you upgrade to a newer rsyslog version). +

The set of available counters will be output as a set of syslog messages. This +output is periodic, with the interval being configurable (default is 5 minutes). +Be sure that your configuration records the counter messages (default is syslog.info). +

Note that loading this module has impact on rsyslog performance. Depending on +settings, this impact may be severe (for high-load environments). +

+

Configuration Directives:

+
    +
  • $PStatsInterval <Seconds>
    +Sets the interval, in seconds at which messages are generated. Please note that the +actual interval may be a bit longer. We do not try to be precise and so the interval is +actually a sleep period which is entered after generating all messages. So the actual +interval is what is configured here plus the actual time required to generate messages. +In general, the difference should not really matter. +
  • $PStatsFacility <numerical facility>
    +The numerical syslog facility code to be used for generated messages. Default +is 5 (syslog).This is useful for filtering messages.
    • +
  • $PStatsSeverity <numerical severity>
    +The numerical syslog severity code to be used for generated messages. Default +is 6 (info).This is useful for filtering messages.
    • +
+Caveats/Known Bugs: +
    +
  • experimental code
    • +
+

Sample:

+

This activates the module and records messages to /var/log/rsyslog-stats in 10 minute intervals:
+

+ +

[rsyslog.conf overview] +[manual index] [rsyslog site]

+

This documentation is part of the +rsyslog +project.
+Copyright © 2010 by Rainer +Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

+ diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 2a64461d..85899954 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -44,6 +44,7 @@ to message generators.
  • imuxsock - unix sockets, including the system log socket
  • imsolaris - input for the Sun Solaris system log source
  • im3195 - accepts syslog messages via RFC 3195
    • +
  • impstats - provides periodic statistics of rsyslog internal counters

    • Output Modules

    diff --git a/plugins/impstats/Makefile.am b/plugins/impstats/Makefile.am new file mode 100644 index 00000000..187bbca4 --- /dev/null +++ b/plugins/impstats/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = impstats.la + +impstats_la_SOURCES = impstats.c +impstats_la_CPPFLAGS = $(RSRT_CFLAGS) -I$(top_srcdir) $(PTHREADS_CFLAGS) +impstats_la_LDFLAGS = -module -avoid-version +impstats_la_LIBADD = diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c new file mode 100644 index 00000000..5f0d19eb --- /dev/null +++ b/plugins/impstats/impstats.c @@ -0,0 +1,211 @@ +/* impstats.c + * A module to periodically output statistics gathered by rsyslog. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c) + * This file is under development and has not yet arrived at being fully + * self-contained and a real object. So far, it is mostly an excerpt + * of the "old" message code without any modifications. However, it + * helps to have things at the right place one we go to the meat of it. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "cfsysline.h" +#include "module-template.h" +#include "errmsg.h" +#include "msg.h" +#include "srUtils.h" +#include "unicode-helper.h" +#include "glbl.h" +#include "prop.h" + +MODULE_TYPE_INPUT + +/* defines */ +#define DEFAULT_STATS_PERIOD (5 * 60) +#define DEFAULT_FACILITY 5 /* syslog */ +#define DEFAULT_SEVERITY 6 /* info */ + +/* Module static data */ +DEF_IMOD_STATIC_DATA +DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) +typedef struct configSettings_s { + int iStatsInterval; + int iFacility; + int iSeverity; +} configSettings_t; + +static configSettings_t cs; + +static prop_t *pInputName = NULL; +static prop_t *pLocalHostIP = NULL; + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +static inline void +initConfigSettings(void) +{ + cs.iStatsInterval = DEFAULT_STATS_PERIOD; + cs.iFacility = DEFAULT_FACILITY; + cs.iSeverity = DEFAULT_SEVERITY; +} + + +/* actually submit a message to the rsyslog core + */ +static inline rsRetVal +doSubmitMsg(uchar *line) +{ + msg_t *pMsg; + DEFiRet; + + CHKiRet(msgConstruct(&pMsg)); + MsgSetInputName(pMsg, pInputName); + MsgSetRawMsgWOSize(pMsg, (char*)line); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); + MsgSetRcvFromIP(pMsg, pLocalHostIP); + MsgSetMSGoffs(pMsg, 0); + MsgSetTAG(pMsg, UCHAR_CONSTANT("rsyslogd-pstats:"), sizeof("rsyslogd-pstats:") - 1); + pMsg->iFacility = cs.iFacility; + pMsg->iSeverity = cs.iSeverity; + pMsg->msgFlags = 0; + + submitMsg(pMsg); + +finalize_it: + RETiRet; + +} + + +/* the function to generate the actual statistics messages + * rgerhards, 2010-09-09 + */ +static inline void +generateStatsMsgs(void) +{ + int iMsgQueueSize; + uchar msg[1024]; + rsRetVal iRet; + + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + snprintf((char*)msg, sizeof(msg), "mainqueue size=%d", iMsgQueueSize); + doSubmitMsg(msg); + +finalize_it: + /*empty statement needed per C syntax*/; +} + + +BEGINrunInput +CODESTARTrunInput + /* this is an endless loop - it is terminated when the thread is + * signalled to do so. This, however, is handled by the framework, + * right into the sleep below. + */ + while(1) { +dbgprintf("impstats: stats interval %d seconds - going to sleep\n", cs.iStatsInterval); + srSleep(cs.iStatsInterval, 0); /* seconds, micro seconds */ +dbgprintf("impstats: woke up\n"); + + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + + generateStatsMsgs(); + } +ENDrunInput + + +BEGINwillRun +CODESTARTwillRun + DBGPRINTF("impstats: stats interval %d seconds\n", cs.iStatsInterval); + if(cs.iStatsInterval == 0) + ABORT_FINALIZE(RS_RET_NO_RUN); +finalize_it: +ENDwillRun + + +BEGINafterRun +CODESTARTafterRun +ENDafterRun + + +BEGINmodExit +CODESTARTmodExit + /* release objects we used */ + objRelease(glbl, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); +ENDmodExit + + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + initConfigSettings(); + return RS_RET_OK; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + DBGPRINTF("impstats version %s loading\n", VERSION); + initConfigSettings(); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatsinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatfacility", 0, eCmdHdlrInt, NULL, &cs.iFacility, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatseverity", 0, eCmdHdlrInt, NULL, &cs.iSeverity, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("rsyslogd-pstats"), sizeof("rsyslogd-pstats") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + + CHKiRet(prop.Construct(&pLocalHostIP)); + CHKiRet(prop.SetString(pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pLocalHostIP)); +ENDmodInit +/* vi:set ai: + */ -- cgit v1.2.3 From 3d50fd6a67b7d0bf31628aea14128a47dd5d326f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Sep 2010 12:37:11 +0200 Subject: fixed small memory leak, corrected input name --- plugins/impstats/impstats.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index 5f0d19eb..a818d20c 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -166,6 +166,8 @@ ENDafterRun BEGINmodExit CODESTARTmodExit + prop.Destruct(&pInputName); + prop.Destruct(&pLocalHostIP); /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); @@ -200,7 +202,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(prop.Construct(&pInputName)); - CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("rsyslogd-pstats"), sizeof("rsyslogd-pstats") - 1)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("impstats"), sizeof("impstats") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); CHKiRet(prop.Construct(&pLocalHostIP)); -- cgit v1.2.3 From e86cb62f1299ef18732f7b3b87d45a840ee38f1e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 13 Sep 2010 15:43:56 +0200 Subject: improved statistics-gathering subsystem ... well, actually this is a first real implementation of this subsystem. I have added a counter registry, a way to access the countres (as readable string) and a way to define and maintem them. Also, module impstats has been updated to utilize the new system. Finally, I added some counters. I hope that this sets the baseline for useful future enhancements. --- doc/impstats.html | 2 + plugins/impstats/impstats.c | 39 ++++-- plugins/imuxsock/imuxsock.c | 18 +++ runtime/Makefile.am | 6 +- runtime/queue.c | 32 +++++ runtime/queue.h | 6 + runtime/rsyslog.c | 3 + runtime/rsyslog.h | 2 + runtime/statsobj.c | 315 ++++++++++++++++++++++++++++++++++++++++++++ runtime/statsobj.h | 124 +++++++++++++++++ 10 files changed, 533 insertions(+), 14 deletions(-) create mode 100644 runtime/statsobj.c create mode 100644 runtime/statsobj.h diff --git a/doc/impstats.html b/doc/impstats.html index 0f4096ab..3b4191e8 100644 --- a/doc/impstats.html +++ b/doc/impstats.html @@ -37,6 +37,8 @@ is 6 (info).This is useful for filtering messages. Caveats/Known Bugs:
      +
    • This module MUST be loaded right at the top of rsyslog.conf, otherwise +stats may not get turned on in all places.
    • experimental code

    Sample:

    diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index a818d20c..1312a4e8 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -45,6 +45,7 @@ #include "srUtils.h" #include "unicode-helper.h" #include "glbl.h" +#include "statsobj.h" #include "prop.h" MODULE_TYPE_INPUT @@ -58,6 +59,9 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(statsobj) +DEFobjCurrIf(errmsg) + typedef struct configSettings_s { int iStatsInterval; int iFacility; @@ -112,22 +116,25 @@ finalize_it: } +/* callback for statsobj + * Note: usrptr exists only to satisfy requirements of statsobj callback interface! + */ +static rsRetVal +doStatsLine(void __attribute__((unused)) *usrptr, cstr_t *cstr) +{ + DEFiRet; + doSubmitMsg(rsCStrGetSzStrNoNULL(cstr)); + RETiRet; +} + + /* the function to generate the actual statistics messages * rgerhards, 2010-09-09 */ static inline void generateStatsMsgs(void) { - int iMsgQueueSize; - uchar msg[1024]; - rsRetVal iRet; - - CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); - snprintf((char*)msg, sizeof(msg), "mainqueue size=%d", iMsgQueueSize); - doSubmitMsg(msg); - -finalize_it: - /*empty statement needed per C syntax*/; + statsobj.GetAllStatsLines(doStatsLine, NULL); } @@ -138,9 +145,7 @@ CODESTARTrunInput * right into the sleep below. */ while(1) { -dbgprintf("impstats: stats interval %d seconds - going to sleep\n", cs.iStatsInterval); srSleep(cs.iStatsInterval, 0); /* seconds, micro seconds */ -dbgprintf("impstats: woke up\n"); if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ @@ -151,10 +156,16 @@ ENDrunInput BEGINwillRun + rsRetVal localRet; CODESTARTwillRun DBGPRINTF("impstats: stats interval %d seconds\n", cs.iStatsInterval); if(cs.iStatsInterval == 0) ABORT_FINALIZE(RS_RET_NO_RUN); + localRet = statsobj.EnableStats(); + if(localRet != RS_RET_OK) { + errmsg.LogError(0, localRet, "impstat: error enabling statistics gathering"); + ABORT_FINALIZE(RS_RET_NO_RUN); + } finalize_it: ENDwillRun @@ -171,6 +182,8 @@ CODESTARTmodExit /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); ENDmodExit @@ -196,6 +209,8 @@ CODEmodInit_QueryRegCFSLineHdlr initConfigSettings(); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatsinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatfacility", 0, eCmdHdlrInt, NULL, &cs.iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatseverity", 0, eCmdHdlrInt, NULL, &cs.iSeverity, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index db53fcb6..d976c676 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -48,6 +48,7 @@ #include "debug.h" #include "unlimited_select.h" #include "sd-daemon.h" +#include "statsobj.h" MODULE_TYPE_INPUT @@ -72,6 +73,10 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(statsobj) + +statsobj_t *modStats; +STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ @@ -269,6 +274,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); + STATSCOUNTER_INC(ctrSubmit, mutCtrSubmit); finalize_it: RETiRet; } @@ -453,9 +459,12 @@ ENDafterRun BEGINmodExit CODESTARTmodExit + statsobj.Destruct(&modStats); + objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); ENDmodExit @@ -502,6 +511,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); @@ -547,6 +557,14 @@ CODEmodInit_QueryRegCFSLineHdlr setSystemLogTimestampIgnore, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&modStats)); + CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), + ctrType_IntCtr, &ctrSubmit)); + CHKiRet(statsobj.ConstructFinalize(modStats)); + ENDmodInit /* vim:set ai: */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 5b2598b2..01b224e7 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -45,6 +45,8 @@ librsyslog_la_SOURCES = \ modules.h \ apc.c \ apc.h \ + statsobj.c \ + statsobj.h \ sync.c \ sync.h \ expr.c \ @@ -81,8 +83,8 @@ librsyslog_la_SOURCES = \ prop.h \ cfsysline.c \ cfsysline.h \ - sd-daemon.c \ - sd-daemon.h \ + sd-daemon.c \ + sd-daemon.h \ \ \ ../action.h \ diff --git a/runtime/queue.c b/runtime/queue.c index 60d17086..387c15c2 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -58,6 +58,7 @@ #include "errmsg.h" #include "datetime.h" #include "unicode-helper.h" +#include "statsobj.h" #include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS @@ -70,6 +71,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(strm) DEFobjCurrIf(errmsg) DEFobjCurrIf(datetime) +DEFobjCurrIf(statsobj) /* forward-definitions */ static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); @@ -1817,6 +1819,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; uchar pszBuf[64]; + uchar *qName; size_t lenBuf; ASSERT(pThis != NULL); @@ -1886,6 +1889,27 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ qqueueAdviseMaxWorkers(pThis); pThis->bQueueStarted = 1; + /* support statistics gathering */ + qName = obj.GetName((obj_t*)pThis); + CHKiRet(statsobj.Construct(&pThis->statsobj)); + CHKiRet(statsobj.SetName(pThis->statsobj, qName)); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("size"), + ctrType_Int, &pThis->iQueueSize)); + + STATSCOUNTER_INIT(pThis->ctrEnqueued, pThis->mutCtrEnqueued); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("enqueued"), + ctrType_IntCtr, &pThis->ctrEnqueued)); + + STATSCOUNTER_INIT(pThis->ctrFull, pThis->mutCtrFull); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("full"), + ctrType_IntCtr, &pThis->ctrFull)); + + pThis->ctrMaxqsize = 0; + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("maxqsize"), + ctrType_Int, &pThis->ctrMaxqsize)); + + CHKiRet(statsobj.ConstructFinalize(pThis->statsobj)); + finalize_it: RETiRet; } @@ -2119,6 +2143,10 @@ CODESTARTobjDestruct(qqueue) free(pThis->pszFilePrefix); free(pThis->pszSpoolDir); + + /* some queues do not provide stats and thus have no statsobj! */ + if(pThis->statsobj != NULL) + statsobj.Destruct(&pThis->statsobj); ENDobjDestruct(qqueue) @@ -2178,6 +2206,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) DEFiRet; struct timespec t; + STATSCOUNTER_INC(pThis->ctrEnqueued, pThis->mutCtrEnqueued); /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); @@ -2225,6 +2254,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); timeoutComp(&t, pThis->toEnq); + STATSCOUNTER_INC(pThis->ctrFull, pThis->mutCtrFull); // TODO : handle enqOnly => discard! if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); @@ -2235,6 +2265,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); + STATSCOUNTER_SETMAX_NOMUT(pThis->ctrMaxqsize, pThis->iQueueSize); finalize_it: RETiRet; @@ -2414,6 +2445,7 @@ BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/queue.h b/runtime/queue.h index 1c758134..38e248cd 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -29,6 +29,7 @@ #include "wtp.h" #include "batch.h" #include "stream.h" +#include "statsobj.h" /* support for the toDelete list */ typedef struct toDeleteLst_s toDeleteLst_t; @@ -165,6 +166,11 @@ struct queue_s { } tVars; DEF_ATOMIC_HELPER_MUT(mutQueueSize); DEF_ATOMIC_HELPER_MUT(mutLogDeq); + /* for statistics subsystem */ + statsobj_t *statsobj; + STATSCOUNTER_DEF(ctrEnqueued, mutCtrEnqueued); + STATSCOUNTER_DEF(ctrFull, mutCtrFull); + int ctrMaxqsize; }; diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index a9794840..8baa2b59 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -82,6 +82,7 @@ #include "ruleset.h" #include "parser.h" #include "strgen.h" +#include "statsobj.h" #include "atomic.h" /* forward definitions */ @@ -150,6 +151,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) * class immediately after it is initialized. And, of course, we load those classes * first that we use ourselfs... -- rgerhards, 2008-03-07 */ + if(ppErrObj != NULL) *ppErrObj = "statsobj"; + CHKiRet(statsobjClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "prop"; CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 34eaedca..7b2655b0 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -150,6 +150,8 @@ typedef struct parser_s parser_t; typedef struct parserList_s parserList_t; typedef struct strgen_s strgen_t; typedef struct strgenList_s strgenList_t; +typedef struct statsobj_s statsobj_t; +typedef struct statsctr_s statsctr_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ diff --git a/runtime/statsobj.c b/runtime/statsobj.c new file mode 100644 index 00000000..e1a89cf4 --- /dev/null +++ b/runtime/statsobj.c @@ -0,0 +1,315 @@ +/* The statsobj object. + * + * This object provides a statistics-gathering facility inside rsyslog. This + * functionality will be pragmatically implemented and extended. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include + +#include "rsyslog.h" +#include "unicode-helper.h" +#include "obj.h" +#include "statsobj.h" +#include "sysvar.h" +#include "srUtils.h" +#include "stringbuf.h" + + +/* externally-visiable data (see statsobj.h for explanation) */ +int GatherStats = 0; + +/* static data */ +DEFobjStaticHelpers + +/* doubly linked list of stats objects. Object is automatically linked to it + * upon construction. Enqueue always happens at the front (simplifies logic). + */ +static statsobj_t *objRoot = NULL; +static statsobj_t *objLast = NULL; + +static pthread_mutex_t mutStats; + +/* ------------------------------ statsobj linked list maintenance ------------------------------ */ + +static inline void +addToObjList(statsobj_t *pThis) +{ + pthread_mutex_lock(&mutStats); + pThis->prev = objLast; + if(objLast != NULL) + objLast->next = pThis; + objLast = pThis; + if(objRoot == NULL) + objRoot = pThis; + pthread_mutex_unlock(&mutStats); +} + + +static inline void +removeFromObjList(statsobj_t *pThis) +{ + pthread_mutex_lock(&mutStats); + if(pThis->prev != NULL) + pThis->prev->next = pThis->next; + if(pThis->next != NULL) + pThis->next->prev = pThis->prev; + if(objLast == pThis) + objLast = pThis->prev; + if(objRoot == pThis) + objRoot = pThis->next; + pthread_mutex_unlock(&mutStats); +} + + +static inline void +addCtrToList(statsobj_t *pThis, ctr_t *pCtr) +{ + pthread_mutex_lock(&pThis->mutCtr); + pCtr->prev = pThis->ctrLast; + if(pThis->ctrLast != NULL) + pThis->ctrLast->next = pCtr; + pThis->ctrLast = pCtr; + if(pThis->ctrRoot == NULL) + pThis->ctrRoot = pCtr; + pthread_mutex_unlock(&pThis->mutCtr); +} + +/* ------------------------------ methods ------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(statsobj) /* be sure to specify the object type also in END macro! */ + pthread_mutex_init(&pThis->mutCtr, NULL); + pThis->ctrLast = NULL; + pThis->ctrRoot = NULL; +ENDobjConstruct(statsobj) + + +/* ConstructionFinalizer + */ +static rsRetVal +statsobjConstructFinalize(statsobj_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, statsobj); + addToObjList(pThis); + RETiRet; +} + + +/* set name. Note that we make our own copy of the memory, caller is + * responsible to free up name it passes in (if required). + */ +static rsRetVal +setName(statsobj_t *pThis, uchar *name) +{ + DEFiRet; + CHKmalloc(pThis->name = ustrdup(name)); +finalize_it: + RETiRet; +} + + +/* add a counter to an object + * ctrName is duplicated, caller must free it if requried + */ +static rsRetVal +addCounter(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr) +{ + ctr_t *ctr; + DEFiRet; + + CHKmalloc(ctr = malloc(sizeof(ctr_t))); + ctr->next = NULL; + ctr->prev = NULL; + CHKmalloc(ctr->name = ustrdup(ctrName)); + ctr->ctrType = ctrType; + switch(ctrType) { + case ctrType_IntCtr: + ctr->val.pIntCtr = (intctr_t*) pCtr; + break; + case ctrType_Int: + ctr->val.pInt = (int*) pCtr; + break; + } + addCtrToList(pThis, ctr); + +finalize_it: + RETiRet; +} + + +/* get all the object's countes together with object name as one line. + */ +static rsRetVal +getStatsLine(statsobj_t *pThis, cstr_t **ppcstr) +{ + cstr_t *pcstr; + ctr_t *pCtr; + DEFiRet; + + CHKiRet(cstrConstruct(&pcstr)); + rsCStrAppendStr(pcstr, pThis->name); + rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT(": "), 2); + + /* now add all counters to this line */ + pthread_mutex_lock(&pThis->mutCtr); + for(pCtr = pThis->ctrRoot ; pCtr != NULL ; pCtr = pCtr->next) { + rsCStrAppendStr(pcstr, pCtr->name); + cstrAppendChar(pcstr, '='); + switch(pCtr->ctrType) { + case ctrType_IntCtr: + rsCStrAppendInt(pcstr, *(pCtr->val.pIntCtr)); // TODO: OK????? + break; + case ctrType_Int: + rsCStrAppendInt(pcstr, *(pCtr->val.pInt)); + break; + } + cstrAppendChar(pcstr, ' '); + } + pthread_mutex_unlock(&pThis->mutCtr); + + CHKiRet(cstrFinalize(pcstr)); + *ppcstr = pcstr; + +finalize_it: + RETiRet; +} + + +/* this function can be used to obtain all stats lines. In this case, + * a callback must be provided. This module than iterates over all objects and + * submits each stats line to the callback. The callback has two parameters: + * the first one is a caller-provided void*, the second one the cstr_t with the + * line. If the callback reports an error, processing is stopped. + */ +static rsRetVal +getAllStatsLines(rsRetVal(*cb)(void*, cstr_t*), void *usrptr) +{ + statsobj_t *o; + cstr_t *cstr; + DEFiRet; + + for(o = objRoot ; o != NULL ; o = o->next) { + CHKiRet(getStatsLine(o, &cstr)); + CHKiRet(cb(usrptr, cstr)); + rsCStrDestruct(&cstr); + } + +finalize_it: + RETiRet; +} + + +/* Enable statistics gathering. currently there is no function to disable it + * again, as this is right now not needed. + */ +static rsRetVal +enableStats() +{ + GatherStats = 1; + return RS_RET_OK; +} + + +/* destructor for the statsobj object */ +BEGINobjDestruct(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ + ctr_t *ctr, *ctrToDel; +CODESTARTobjDestruct(statsobj) + removeFromObjList(pThis); + + /* destruct counters */ + ctr = pThis->ctrRoot; + while(ctr != NULL) { + ctrToDel = ctr; + ctr = ctr->next; + free(ctrToDel->name); + free(ctrToDel); + } + + pthread_mutex_destroy(&pThis->mutCtr); + free(pThis->name); +ENDobjDestruct(statsobj) + + +/* debugprint for the statsobj object */ +BEGINobjDebugPrint(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(statsobj) + dbgoprint((obj_t*) pThis, "statsobj object, currently no state info available\n"); +ENDobjDebugPrint(statsobj) + + +/* queryInterface function + */ +BEGINobjQueryInterface(statsobj) +CODESTARTobjQueryInterface(statsobj) + if(pIf->ifVersion != statsobjCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = statsobjConstruct; + pIf->ConstructFinalize = statsobjConstructFinalize; + pIf->Destruct = statsobjDestruct; + pIf->DebugPrint = statsobjDebugPrint; + pIf->SetName = setName; + pIf->GetStatsLine = getStatsLine; + pIf->GetAllStatsLines = getAllStatsLines; + pIf->AddCounter = addCounter; + pIf->EnableStats = enableStats; +finalize_it: +ENDobjQueryInterface(statsobj) + + +/* Initialize the statsobj class. Must be called as the very first method + * before anything else is called inside this class. + */ +BEGINAbstractObjClassInit(statsobj, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, statsobjDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, statsobjConstructFinalize); + + /* init other data items */ + pthread_mutex_init(&mutStats, NULL); + +ENDObjClassInit(statsobj) + +/* Exit the class. + */ +BEGINObjClassExit(statsobj, OBJ_IS_CORE_MODULE) /* class, version */ + /* release objects we no longer need */ + pthread_mutex_destroy(&mutStats); +ENDObjClassExit(statsobj) diff --git a/runtime/statsobj.h b/runtime/statsobj.h new file mode 100644 index 00000000..1ab02182 --- /dev/null +++ b/runtime/statsobj.h @@ -0,0 +1,124 @@ +/* The statsobj object. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STATSOBJ_H +#define INCLUDED_STATSOBJ_H + +#include "atomic.h" + +/* The following data item is somewhat dirty, in that it does not follow + * our usual object calling conventions. However, much like with "Debug", we + * do this to gain speed. If we finally come to a platform that does not + * provide resolution of names for dynamically loaded modules, we need to find + * a work-around, but until then, we use the direct access. + * If set to 0, statistics are not gathered, otherwise they are. + */ +extern int GatherStats; + +/* our basic counter type -- need 32 bit on 32 bit platform. + * IMPORTANT: this type *MUST* be supported by atomic instructions! + */ +typedef uint64 intctr_t; + +/* counter types */ +typedef enum statsCtrType_e { + ctrType_IntCtr, + ctrType_Int +} statsCtrType_t; + + +/* helper entity, the counter */ +typedef struct ctr_s { + uchar *name; + statsCtrType_t ctrType; + union { + intctr_t *pIntCtr; + int *pInt; + } val; + struct ctr_s *next, *prev; +} ctr_t; + +/* the statsobj object */ +struct statsobj_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *name; + pthread_mutex_t mutCtr; /* to guard counter linked-list ops */ + ctr_t *ctrRoot; /* doubly-linked list of statsobj counters */ + ctr_t *ctrLast; + /* used to link ourselves together */ + statsobj_t *prev; + statsobj_t *next; +}; + + +/* interfaces */ +BEGINinterface(statsobj) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(statsobj); + rsRetVal (*Construct)(statsobj_t **ppThis); + rsRetVal (*ConstructFinalize)(statsobj_t *pThis); + rsRetVal (*Destruct)(statsobj_t **ppThis); + rsRetVal (*SetName)(statsobj_t *pThis, uchar *name); + rsRetVal (*GetStatsLine)(statsobj_t *pThis, cstr_t **ppcstr); + rsRetVal (*GetAllStatsLines)(rsRetVal(*cb)(void*, cstr_t*), void *usrptr); + rsRetVal (*AddCounter)(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr); + rsRetVal (*EnableStats)(void); +ENDinterface(statsobj) +#define statsobjCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(statsobj); + + +/* macros to handle stats counters + * These are to be used by "counter providers". Note that we MUST + * specify the mutex name, even though at first it looks like it + * could be automatically be generated via e.g. "mut##ctr". + * Unfortunately, this does not work if counter is e.g. "pThis->ctr". + * So we decided, for clarity, to always insist on specifying the mutex + * name (after all, it's just a few more keystrokes...). + */ +#define STATSCOUNTER_DEF(ctr, mut) \ + intctr_t ctr; \ + DEF_ATOMIC_HELPER_MUT(mut) + +#define STATSCOUNTER_INIT(ctr, mut) \ + INIT_ATOMIC_HELPER_MUT(mut); \ + ctr = 0; + +#define STATSCOUNTER_INC(ctr, mut) \ + if(GatherStats) \ + ATOMIC_INC(&ctr, mut); + +#define STATSCOUNTER_DEC(ctr, mut) \ + if(GatherStats) \ + ATOMIC_DEC(&ctr, mut); + +/* the next macro works only if the variable is already guarded + * by mutex (or the users risks a wrong result). It is assumed + * that there are not concurrent operations that modify the counter. + */ +#define STATSCOUNTER_SETMAX_NOMUT(ctr, newmax) \ + if(GatherStats && ((newmax) > (ctr))) \ + ctr = newmax; + +#endif /* #ifndef INCLUDED_STATSOBJ_H */ -- cgit v1.2.3 From 6173abfbc9f59c7a5551cb6713ff182e68165f71 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Sep 2010 15:41:31 +0200 Subject: preparing for 5.7.0 --- ChangeLog | 2 +- doc/manual.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0f8b625d..9696f398 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-?? +Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters - support for systemd officially added * acquire /dev/log socket optionally from systemd diff --git a/doc/manual.html b/doc/manual.html index 882af031..08334c5d 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.5.7 (beta branch) of rsyslog. +

    This documentation is for version 5.7.0 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 2288bbb90fc6f51cebf5db81b05c6141e754ef25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Sep 2010 15:58:03 +0200 Subject: fixed compile problems on Solaris --- runtime/statsobj.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/statsobj.h b/runtime/statsobj.h index 1ab02182..7447cded 100644 --- a/runtime/statsobj.h +++ b/runtime/statsobj.h @@ -99,7 +99,7 @@ PROTOTYPEObj(statsobj); */ #define STATSCOUNTER_DEF(ctr, mut) \ intctr_t ctr; \ - DEF_ATOMIC_HELPER_MUT(mut) + DEF_ATOMIC_HELPER_MUT(mut); #define STATSCOUNTER_INIT(ctr, mut) \ INIT_ATOMIC_HELPER_MUT(mut); \ @@ -107,7 +107,7 @@ PROTOTYPEObj(statsobj); #define STATSCOUNTER_INC(ctr, mut) \ if(GatherStats) \ - ATOMIC_INC(&ctr, mut); + ATOMIC_INC(&ctr, &mut); #define STATSCOUNTER_DEC(ctr, mut) \ if(GatherStats) \ -- cgit v1.2.3 From 735be1e247563d422c18d955530eb8360e706361 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Sep 2010 10:15:20 +0200 Subject: minor: improved error message when already running --- tools/syslogd.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index 56000e08..4964f8fc 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2437,10 +2437,9 @@ doGlblProcessInit(void) for (i = 3; i < num_fds; i++) (void) close(i); untty(); - } - else - { - fputs(" Already running.\n", stderr); + } else { + fputs(" Already running. If you want to run multiple instances, you need " + "to specify different pid files (use -i option)\n", stderr); exit(1); /* "good" exit, done if syslogd is already running */ } } -- cgit v1.2.3 From 2a2cc3d26d56b834dcc143f2f4e3abbb6bd206c6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Sep 2010 12:20:59 +0200 Subject: minor: added version number to some error messages --- runtime/debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/debug.c b/runtime/debug.c index 64e251e5..5818dda9 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1310,7 +1310,7 @@ dbgGetRuntimeOptions(void) while(dbgGetRTOptNamVal(&pszOpts, &optname, &optval)) { if(!strcasecmp((char*)optname, "help")) { fprintf(stderr, - "rsyslogd runtime debug support - help requested, rsyslog terminates\n\n" + "rsyslogd " VERSION " runtime debug support - help requested, rsyslog terminates\n\n" "environment variables:\n" "addional logfile: export RSYSLOG_DEBUGFILE=\"/path/to/file\"\n" "to set: export RSYSLOG_DEBUG=\"cmd cmd cmd\"\n\n" @@ -1358,7 +1358,7 @@ dbgGetRuntimeOptions(void) bAbortTrace = 0; } else if(!strcasecmp((char*)optname, "filetrace")) { if(*optval == '\0') { - fprintf(stderr, "Error: logfile debug option requires filename, " + fprintf(stderr, "rsyslogd " VERSION " error: logfile debug option requires filename, " "e.g. \"logfile=debug.c\"\n"); exit(1); } else { @@ -1366,7 +1366,7 @@ dbgGetRuntimeOptions(void) dbgPrintNameAdd(optval, &printNameFileRoot); } } else { - fprintf(stderr, "Error: invalid debug option '%s', value '%s' - ignored\n", + fprintf(stderr, "rsyslogd " VERSION " error: invalid debug option '%s', value '%s' - ignored\n", optval, optname); } } -- cgit v1.2.3 From aa1b8f38c3611fd01710e2e3d0189c90cf9a7e9d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 18:05:12 +0200 Subject: first shot at SCM_CREDENTIALS --- plugins/imuxsock/imuxsock.c | 67 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index d976c676..9a245f88 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "dirty.h" #include "cfsysline.h" #include "unicode-helper.h" @@ -192,6 +193,7 @@ static int create_unix_socket(const char *path, int bCreatePath) { struct sockaddr_un sunx; int fd; + int one; if (path[0] == '\0') return -1; @@ -226,6 +228,7 @@ static int create_unix_socket(const char *path, int bCreatePath) return -1; } + //TOD: add logic to prepare socket (or at better place!) return fd; } } @@ -246,6 +249,24 @@ static int create_unix_socket(const char *path, int bCreatePath) close(fd); return -1; } + one = 1; +dbgprintf("imuxsock: setting socket options!\n"); + if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + close(fd); + return -1; + } + if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + close(fd); + return -1; + } + one = 1; + if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); + close(fd); + return -1; + } return fd; } @@ -264,6 +285,12 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + // TODO: here we need to mangle the raw message if we need to + // "fix up" the user pid. In the long term, it may make more sense + // to add this (somehow) to the message object (problem: at this stage, + // it is not fully available, eg no structured data). Alternatively, we + // may use a custom parser for our messages, but that doesn't play too well + // with the rest of the system. if(funixParseHost[iSock]) { pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; } else { @@ -292,7 +319,13 @@ static rsRetVal readSocket(int fd, int iSock) DEFiRet; int iRcvd; int iMaxLine; + struct msghdr msgh; + struct iovec msgiov; + static int ratelimitErrmsg = 0; // TODO: atomic OPS + struct cmsghdr *cm; + struct ucred *cred; uchar bufRcv[4096+1]; + char aux[1024]; uchar *pRcv = NULL; /* receive buffer */ assert(iSock >= 0); @@ -311,15 +344,43 @@ static rsRetVal readSocket(int fd, int iSock) CHKmalloc(pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))); } - iRcvd = recv(fd, pRcv, iMaxLine, 0); + memset(&msgh, 0, sizeof(msgh)); + memset(&msgiov, 0, sizeof(msgiov)); + memset(&aux, 0, sizeof(aux)); + msgiov.iov_base = pRcv; + msgiov.iov_len = iMaxLine; + msgh.msg_iov = &msgiov; + msgh.msg_iovlen = 1; + msgh.msg_control = aux; + msgh.msg_controllen = sizeof(aux); + iRcvd = recvmsg(fd, &msgh, MSG_DONTWAIT); +/* iRcvd = recv(fd, pRcv, iMaxLine, 0); + */ + dbgprintf("Message from UNIX socket: #%d\n", fd); if (iRcvd > 0) { + ratelimitErrmsg = 0; + + +dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); + for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { +dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { + cred = (struct ucred*) CMSG_DATA(cm); + dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); + //break; + } + } +dbgprintf("XXX: post CM loop\n"); + + + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); - } else if (iRcvd < 0 && errno != EINTR) { + } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("UNIX socket error: %d = %s.\n", errno, errStr); - errmsg.LogError(errno, NO_ERRCODE, "recvfrom UNIX"); + errmsg.LogError(errno, NO_ERRCODE, "imuxsock: recvfrom UNIX"); } finalize_it: -- cgit v1.2.3 From c2d36c6894f7c36c2aed23c1e7ee9580c58659a7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 18:57:05 +0200 Subject: patching SCM_CREDENTIALS pid into TAG of imuxsock read messages --- plugins/imuxsock/imuxsock.c | 179 ++++++++++++++++++++++++++++++++++---------- runtime/rsyslog.h | 1 + 2 files changed, 142 insertions(+), 38 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 9a245f88..fff6ac72 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -29,6 +29,7 @@ #include "rsyslog.h" #include #include +#include #include #include #include @@ -50,6 +51,7 @@ #include "unlimited_select.h" #include "sd-daemon.h" #include "statsobj.h" +#include "datetime.h" MODULE_TYPE_INPUT @@ -74,6 +76,7 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(datetime) DEFobjCurrIf(statsobj) statsobj_t *modStats; @@ -189,9 +192,38 @@ static rsRetVal discardFunixn(void) } -static int create_unix_socket(const char *path, int bCreatePath) +/* used to create a log socket if NOT passed in via systemd. + */ +static inline rsRetVal +createLogSocket(const char *path, int bCreatePath, int *fd) { struct sockaddr_un sunx; + DEFiRet; + + unlink(path); + memset(&sunx, 0, sizeof(sunx)); + sunx.sun_family = AF_UNIX; + if(bCreatePath) { + makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); + } + strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); + *fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if(*fd < 0 || bind(*fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || + chmod(path, 0666) < 0) { + errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); + dbgprintf("cannot create %s (%d).\n", path, errno); + close(*fd); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } +finalize_it: + RETiRet; +} + + +static inline rsRetVal +openLogSocket(const char *path, int bCreatePath, int *pfd) +{ + DEFiRet; int fd; int one; @@ -201,18 +233,18 @@ static int create_unix_socket(const char *path, int bCreatePath) if (strcmp(path, _PATH_LOG) == 0) { int r; - /* Check whether an FD was passed in from systemd. If + /* System log socket code. Check whether an FD was passed in from systemd. If * so, it's the /dev/log socket, so use it. */ r = sd_listen_fds(0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to acquire systemd socket"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (r > 1) { errmsg.LogError(EINVAL, NO_ERRCODE, "Wrong number of systemd sockets passed"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (r == 1) { @@ -220,46 +252,31 @@ static int create_unix_socket(const char *path, int bCreatePath) r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (!r) { errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } - - //TOD: add logic to prepare socket (or at better place!) - return fd; - } - } - - unlink(path); - - memset(&sunx, 0, sizeof(sunx)); - sunx.sun_family = AF_UNIX; - if(bCreatePath) { - makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); - } - (void) strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); - fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if (fd < 0 || bind(fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || - chmod(path, 0666) < 0) { - errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); - dbgprintf("cannot create %s (%d).\n", path, errno); - close(fd); - return -1; + } else { + CHKiRet(createLogSocket(path, bCreatePath, &fd)); + } + } else { + CHKiRet(createLogSocket(path, bCreatePath, &fd)); } + one = 1; dbgprintf("imuxsock: setting socket options!\n"); - if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } one = 1; if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { @@ -267,16 +284,53 @@ dbgprintf("imuxsock: setting socket options!\n"); close(fd); return -1; } - return fd; + *pfd = fd; +finalize_it: + RETiRet; +} + + +/* patch correct pid into tag. bufTAG MUST be CONF_TAG_MAXSIZE long! + */ +static inline void +fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) +{ + int i; + char bufPID[16]; + int lenPID; + + if(cred == NULL) + return; + + lenPID = snprintf(bufPID, sizeof(bufPID), "[%u]:", (unsigned) cred->pid); + + for(i = *lenTag ; i >= 0 && bufTAG[i] != '[' ; --i) + /*JUST SKIP*/; + + if(i < 0) + i = *lenTag - 1; /* go right at end of TAG, pid was not present (-1 for ':') */ + + if(i + lenPID > CONF_TAG_MAXSIZE) + return; /* do not touch, as things would break */ + + memcpy(bufTAG + i, bufPID, lenPID); + *lenTag = i + lenPID; } /* submit received message to the queue engine + * We now parse the message according to expected format so that we + * can also mangle it if necessary. */ static inline rsRetVal -SubmitMsg(uchar *pRcv, int lenRcv, int iSock) +SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) { msg_t *pMsg; + int lenMsg; + int i; + uchar *parse; + int pri; + uchar bufParseTAG[CONF_TAG_MAXSIZE]; DEFiRet; /* we now create our own message object and submit it to the queue */ @@ -285,6 +339,43 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); +// TODO: handle format errors + parse = pRcv; + lenMsg = lenRcv; + + parse++; lenMsg--; /* '<' */ + pri = 0; + while(lenMsg && isdigit(*parse)) { + pri = pri * 10 + *parse - '0'; + ++parse; + --lenMsg; + } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); +dbgprintf("imuxsock: submit: facil %d, sever %d\n", pMsg->iFacility, pMsg->iSeverity); + + parse++; lenMsg--; /* '>' */ + +dbgprintf("imuxsock: submit: stage 2\n"); + if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { + dbgprintf("we have a problem, invalid timestamp in msg!\n"); + } + + /* pull tag */ +dbgprintf("imuxsock: submit: stage 3\n"); + + i = 0; + while(lenMsg > 0 && *parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *parse++; + --lenMsg; + } + bufParseTAG[i] = '\0'; /* terminate string */ + fixPID(bufParseTAG, &i, cred); + MsgSetTAG(pMsg, bufParseTAG, i); + + MsgSetMSGoffs(pMsg, lenRcv - lenMsg); + // TODO: here we need to mangle the raw message if we need to // "fix up" the user pid. In the long term, it may make more sense // to add this (somehow) to the message object (problem: at this stage, @@ -292,9 +383,9 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) // may use a custom parser for our messages, but that doesn't play too well // with the rest of the system. if(funixParseHost[iSock]) { - pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->msgFlags = funixFlags[iSock] | PARSE_HOSTNAME; } else { - pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING; + pMsg->msgFlags = funixFlags[iSock]; } MsgSetRcvFrom(pMsg, funixHName[iSock]); @@ -363,6 +454,7 @@ static rsRetVal readSocket(int fd, int iSock) dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); + cred = NULL; for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { @@ -375,7 +467,7 @@ dbgprintf("XXX: post CM loop\n"); - CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock, cred)); } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -458,6 +550,7 @@ ENDrunInput BEGINwillRun CODESTARTwillRun register int i; + int actSocks; /* first apply some config settings */ # ifdef OS_SOLARIS @@ -474,9 +567,17 @@ CODESTARTwillRun funixn[0] = pLogSockName; /* initialize and return if will run or not */ + actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfunix ; i++) { - if ((funix[i] = create_unix_socket((char*) funixn[i], funixCreateSockPath[i])) != -1) + if(openLogSocket((char*) funixn[i], funixCreateSockPath[i], &(funix[i])) == RS_RET_OK) { + ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); + } + } + + if(actSocks == 0) { + errmsg.LogError(0, NO_ERRCODE, "imuxsock does not run because we could not aquire any socket\n"); + ABORT_FINALIZE(RS_RET_ERR); } /* we need to create the inputName property (only once during our lifetime) */ @@ -526,6 +627,7 @@ CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); objRelease(statsobj, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); ENDmodExit @@ -573,6 +675,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(statsobj, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 7b2655b0..c58c259b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -456,6 +456,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_EPOLL_CR_FAILED = -2173, /**< epoll_create() failed */ RS_RET_EPOLL_CTL_FAILED = -2174, /**< epoll_ctl() failed */ RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ + RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 7d680cb7590d4eefa4631a249a02248b4168c7c6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 19:05:20 +0200 Subject: fixed nit --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index fff6ac72..496e679f 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -282,7 +282,7 @@ dbgprintf("imuxsock: setting socket options!\n"); if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } *pfd = fd; finalize_it: -- cgit v1.2.3 From 43933920353ed6911738255d509cd001960ff1b7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 10:28:47 +0200 Subject: changed imuxsock to have a somewhat better representation of listeners still not great, but far better readable (and extendable) than what we had before. --- plugins/imuxsock/imuxsock.c | 156 +++++++++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 61 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 496e679f..493f11c4 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -82,20 +82,26 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +/* structure to describe a specific listener */ +struct lstn_s { + uchar *sockName; /* read-only after startup */ + prop_t *hostName; /* host-name override - if set, use this instead of actual name */ + int fd; /* read-only after startup */ + int flags; /* should parser parse host name? read-only after startup */ + int flowCtl; /* flow control settings for this socket */ + sbool bParseHost; /* should parser parse host name? read-only after startup */ + sbool bUseCreds; + sbool bCreateSockPath; /* auto-creation of socket directory? */ +}; +static struct lstn_s listeners[MAXFUNIX]; + static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -static int startIndexUxLocalSockets; /* process funix from that index on (used to +static int startIndexUxLocalSockets; /* process fd from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup */ -static int funixParseHost[MAXFUNIX] = { 0, }; /* should parser parse host name? read-only after startup */ -static int funixFlags[MAXFUNIX] = { IGNDATE, }; /* should parser parse host name? read-only after startup */ -static int funixCreateSockPath[MAXFUNIX] = { 0, }; /* auto-creation of socket directory? */ -static uchar *funixn[MAXFUNIX] = { (uchar*) _PATH_LOG }; /* read-only after startup */ -static prop_t *funixHName[MAXFUNIX] = { NULL, }; /* host-name override - if set, use this instead of actual name */ -static int funixFlowCtl[MAXFUNIX] = { eFLOWCTL_NO_DELAY, }; /* flow control settings for this socket */ -static int funix[MAXFUNIX] = { -1, }; /* read-only after startup */ -static int nfunix = 1; /* number of Unix sockets open / read-only after startup */ +static int nfd = 1; /* number of Unix sockets open / read-only after startup */ /* config settings */ static int bOmitLocalLogging = 0; @@ -103,6 +109,7 @@ static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreateSockPath 0 static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? */ @@ -114,7 +121,7 @@ static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? * static rsRetVal setSystemLogTimestampIgnore(void __attribute__((unused)) *pVal, int iNewVal) { DEFiRet; - funixFlags[0] = iNewVal ? IGNDATE : NOFLAG; + listeners[0].flags = iNewVal ? IGNDATE : NOFLAG; RETiRet; } @@ -123,7 +130,7 @@ static rsRetVal setSystemLogTimestampIgnore(void __attribute__((unused)) *pVal, static rsRetVal setSystemLogFlowControl(void __attribute__((unused)) *pVal, int iNewVal) { DEFiRet; - funixFlowCtl[0] = iNewVal ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; + listeners[0].flowCtl = iNewVal ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; RETiRet; } @@ -140,26 +147,28 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; - if(nfunix < MAXFUNIX) { + if(nfd < MAXFUNIX) { if(*pNewVal == ':') { - funixParseHost[nfunix] = 1; + listeners[nfd].bParseHost = 1; } else { - funixParseHost[nfunix] = 0; + listeners[nfd].bParseHost = 0; } - CHKiRet(prop.Construct(&(funixHName[nfunix]))); + CHKiRet(prop.Construct(&(listeners[nfd].hostName))); if(pLogHostName == NULL) { - CHKiRet(prop.SetString(funixHName[nfunix], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.SetString(listeners[nfd].hostName, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); } else { - CHKiRet(prop.SetString(funixHName[nfunix], pLogHostName, ustrlen(pLogHostName))); + CHKiRet(prop.SetString(listeners[nfd].hostName, pLogHostName, ustrlen(pLogHostName))); /* reset hostname for next socket */ free(pLogHostName); pLogHostName = NULL; } - CHKiRet(prop.ConstructFinalize(funixHName[nfunix])); - funixFlowCtl[nfunix] = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; - funixFlags[nfunix] = bIgnoreTimestamp ? IGNDATE : NOFLAG; - funixCreateSockPath[nfunix] = bCreateSockPath; - funixn[nfunix++] = pNewVal; + CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); + listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; + listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; + listeners[nfd].bCreateSockPath = bCreateSockPath; + listeners[nfd].sockName = pNewVal; + listeners[nfd].bUseCreds = bUseCreds; + nfd++; } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", pNewVal); @@ -170,21 +179,21 @@ finalize_it: } -/* free the funixn[] socket names - needed as cleanup on several places - * note that nfunix is NOT reset! funixn[0] is never freed, as it comes from +/* free the sockName[] socket names - needed as cleanup on several places + * note that nfd is NOT reset! sockName[0] is never freed, as it comes from * the constant memory pool - and if not, it is freeed via some other pointer. */ static rsRetVal discardFunixn(void) { int i; - for (i = 1; i < nfunix; i++) { - if(funixn[i] != NULL) { - free(funixn[i]); - funixn[i] = NULL; + for (i = 1; i < nfd; i++) { + if(listeners[i].sockName != NULL) { + free(listeners[i].sockName); + listeners[i].sockName = NULL; } - if(funixHName[i] != NULL) { - prop.Destruct(&(funixHName[i])); + if(listeners[i].hostName != NULL) { + prop.Destruct(&(listeners[i].hostName)); } } @@ -267,7 +276,6 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) } one = 1; -dbgprintf("imuxsock: setting socket options!\n"); if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); @@ -284,6 +292,17 @@ dbgprintf("imuxsock: setting socket options!\n"); close(fd); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } + + + +static socklen_t two =sizeof(one); + if(getsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one,&two) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); + close(fd); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } +dbgprintf("imuxsock: SO_TIMESTAMP is %d\n", one); + *pfd = fd; finalize_it: RETiRet; @@ -337,7 +356,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) CHKiRet(msgConstruct(&pMsg)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); - MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + MsgSetFlowControlType(pMsg, listeners[iSock].flowCtl); // TODO: handle format errors parse = pRcv; @@ -382,13 +401,13 @@ dbgprintf("imuxsock: submit: stage 3\n"); // it is not fully available, eg no structured data). Alternatively, we // may use a custom parser for our messages, but that doesn't play too well // with the rest of the system. - if(funixParseHost[iSock]) { - pMsg->msgFlags = funixFlags[iSock] | PARSE_HOSTNAME; + if(listeners[iSock].bParseHost) { + pMsg->msgFlags = listeners[iSock].flags | PARSE_HOSTNAME; } else { - pMsg->msgFlags = funixFlags[iSock]; + pMsg->msgFlags = listeners[iSock].flags; } - MsgSetRcvFrom(pMsg, funixHName[iSock]); + MsgSetRcvFrom(pMsg, listeners[iSock].hostName); CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); @@ -453,7 +472,7 @@ static rsRetVal readSocket(int fd, int iSock) ratelimitErrmsg = 0; -dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); +dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); cred = NULL; for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); @@ -511,10 +530,11 @@ CODESTARTrunInput maxfds = 0; FD_ZERO (pReadfds); /* Copy master connections */ - for (i = startIndexUxLocalSockets; i < nfunix; i++) { - if (funix[i] != -1) { - FD_SET(funix[i], pReadfds); - if (funix[i]>maxfds) maxfds=funix[i]; + for (i = startIndexUxLocalSockets; i < nfd; i++) { + if (listeners[i].fd!= -1) { + FD_SET(listeners[i].fd, pReadfds); + if(listeners[i].fd > maxfds) + maxfds=listeners[i].fd; } } @@ -531,10 +551,10 @@ CODESTARTrunInput if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ - for (i = 0; i < nfunix && nfds > 0; i++) { + for (i = 0; i < nfd && nfds > 0; i++) { if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ - if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) { + if ((fd = listeners[i].fd) != -1 && FD_ISSET(fd, pReadfds)) { readSocket(fd, i); --nfds; /* indicate we have processed one */ } @@ -564,14 +584,15 @@ CODESTARTwillRun startIndexUxLocalSockets = bOmitLocalLogging ? 1 : 0; # endif if(pLogSockName != NULL) - funixn[0] = pLogSockName; + listeners[0].sockName = pLogSockName; /* initialize and return if will run or not */ actSocks = 0; - for (i = startIndexUxLocalSockets ; i < nfunix ; i++) { - if(openLogSocket((char*) funixn[i], funixCreateSockPath[i], &(funix[i])) == RS_RET_OK) { + for (i = startIndexUxLocalSockets ; i < nfd ; i++) { + if(openLogSocket((char*) listeners[i].sockName, listeners[i].bCreateSockPath, + &(listeners[i].fd)) == RS_RET_OK) { ++actSocks; - dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); + dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } } @@ -594,9 +615,9 @@ CODESTARTafterRun int i; /* do cleanup here */ /* Close the UNIX sockets. */ - for (i = 0; i < nfunix; i++) - if (funix[i] != -1) - close(funix[i]); + for (i = 0; i < nfd; i++) + if (listeners[i].fd != -1) + close(listeners[i].fd); /* Clean-up files. If systemd passed us a socket it is * systemd's job to clean it up.*/ @@ -604,15 +625,15 @@ CODESTARTafterRun i = 1; else i = startIndexUxLocalSockets; - for(; i < nfunix; i++) - if (funixn[i] && funix[i] != -1) - unlink((char*) funixn[i]); + for(; i < nfd; i++) + if (listeners[i].sockName && listeners[i].fd != -1) + unlink((char*) listeners[i].sockName); /* free no longer needed string */ free(pLogSockName); free(pLogHostName); discardFunixn(); - nfunix = 1; + nfd = 1; if(pInputName != NULL) prop.Destruct(&pInputName); @@ -657,9 +678,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } discardFunixn(); - nfunix = 1; + nfd = 1; bIgnoreTimestamp = 1; bUseFlowCtl = 0; + bUseCreds = 0; bCreateSockPath = DFLT_bCreateSockPath; return RS_RET_OK; @@ -679,10 +701,20 @@ CODEmodInit_QueryRegCFSLineHdlr dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); - /* initialize funixn[] array */ + /* init system log socket settings */ + listeners[0].flags = IGNDATE; + listeners[0].sockName = UCHAR_CONSTANT(_PATH_LOG); + listeners[0].hostName = NULL; + listeners[0].flowCtl = eFLOWCTL_NO_DELAY; + listeners[0].fd = -1; + listeners[0].bParseHost = 0; + listeners[0].bUseCreds = 0; + listeners[0].bCreateSockPath = 0; + + /* initialize socket names */ for(i = 1 ; i < MAXFUNIX ; ++i) { - funixn[i] = NULL; - funix[i] = -1; + listeners[i].sockName = NULL; + listeners[i].fd = -1; } CHKiRet(prop.Construct(&pLocalHostIP)); @@ -690,9 +722,9 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(prop.ConstructFinalize(pLocalHostIP)); /* now init listen socket zero, the local log socket */ - CHKiRet(prop.Construct(&(funixHName[0]))); - CHKiRet(prop.SetString(funixHName[0], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); - CHKiRet(prop.ConstructFinalize(funixHName[0])); + CHKiRet(prop.Construct(&(listeners[0].hostName))); + CHKiRet(prop.SetString(listeners[0].hostName, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.ConstructFinalize(listeners[0].hostName)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"omitlocallogging", 0, eCmdHdlrBinary, @@ -707,6 +739,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, NULL, &bCreateSockPath, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusespidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit v1.2.3 From 3773aadcdd8008b97bb532f6d29fe29d17b06159 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 11:41:17 +0200 Subject: added support for SCM_CREDENTIALS to imuxsock (now fully working) --- ChangeLog | 7 ++ doc/imuxsock.html | 8 ++ plugins/imuxsock/imuxsock.c | 197 +++++++++++++++++++++----------------------- 3 files changed, 107 insertions(+), 105 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9696f398..6622b67c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,11 @@ --------------------------------------------------------------------------- +Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? +- imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log + socket itself (thanks to Lennart Poettering for the suggestion) +- added new config statements + * $InputUnixListenSocketUsePIDFromSystem + * $SystemLogUsePIDFromSystem +--------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters - support for systemd officially added diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 381374d2..4af2c030 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -40,12 +40,20 @@ the implications. Note that for many systems, turning on flow control does not h
    Ignore timestamps included in the message. Applies to the next socket being added.

  • $InputUnixListenSocketFlowControl [on/off] - specifies if flow control should be applied to the next socket.
    • +
  • $InputUnixListenSocketUsePIDFromSystem [on/off] - specifies if the pid being logged shall +be obtained from the log socket itself. If so, the TAG part of the message is rewritten. +It is recommended to turn this option on, but the default is "off" to keep compatible +with earlier versions of rsyslog. This option was introduced in 5.7.0.
  • $SystemLogSocketIgnoreMsgTimestamp [on/off]
    Ignore timestamps included in the messages, applies to messages received via the system log socket.
  • $OmitLocalLogging (imuxsock) [on/off] -- former -o option
  • $SystemLogSocketName <name-of-socket> -- former -p option
  • $SystemLogFlowControl [on/off] - specifies if flow control should be applied to the system log socket.
    • +
  • $SystemLogUsePIDFromSystem [on/off] - specifies if the pid being logged shall +be obtained from the log socket itself. If so, the TAG part of the message is rewritten. +It is recommended to turn this option on, but the default is "off" to keep compatible +with earlier versions of rsyslog. This option was introduced in 5.7.0.
  • $InputUnixListenSocketCreatePath [on/off] - create directories in the socket path if they do not already exist. They are created with 0755 permissions with the owner being the process under which rsyslogd runs. The default is not to create directories. Keep in mind, though, that rsyslogd always diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 493f11c4..df2b8efc 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -83,7 +83,7 @@ statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) /* structure to describe a specific listener */ -struct lstn_s { +typedef struct lstn_s { uchar *sockName; /* read-only after startup */ prop_t *hostName; /* host-name override - if set, use this instead of actual name */ int fd; /* read-only after startup */ @@ -91,9 +91,9 @@ struct lstn_s { int flowCtl; /* flow control settings for this socket */ sbool bParseHost; /* should parser parse host name? read-only after startup */ sbool bUseCreds; - sbool bCreateSockPath; /* auto-creation of socket directory? */ -}; -static struct lstn_s listeners[MAXFUNIX]; + sbool bCreatePath; /* auto-creation of socket directory? */ +} lstn_t; +static lstn_t listeners[MAXFUNIX]; static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ @@ -108,10 +108,11 @@ static int bOmitLocalLogging = 0; static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ -static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ -static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ -#define DFLT_bCreateSockPath 0 -static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? */ +static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +#define DFLT_bCreatePath 0 +static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ /* set the timestamp ignore / not ignore option for the system @@ -165,7 +166,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; - listeners[nfd].bCreateSockPath = bCreateSockPath; + listeners[nfd].bCreatePath = bCreatePath; listeners[nfd].sockName = pNewVal; listeners[nfd].bUseCreds = bUseCreds; nfd++; @@ -204,24 +205,25 @@ static rsRetVal discardFunixn(void) /* used to create a log socket if NOT passed in via systemd. */ static inline rsRetVal -createLogSocket(const char *path, int bCreatePath, int *fd) +createLogSocket(lstn_t *pLstn) { struct sockaddr_un sunx; DEFiRet; - unlink(path); + unlink((char*)pLstn->sockName); memset(&sunx, 0, sizeof(sunx)); sunx.sun_family = AF_UNIX; - if(bCreatePath) { - makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); + if(pLstn->bCreatePath) { + makeFileParentDirs((uchar*)pLstn->sockName, ustrlen(pLstn->sockName), 0755, -1, -1, 0); } - strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); - *fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if(*fd < 0 || bind(*fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || - chmod(path, 0666) < 0) { - errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); - dbgprintf("cannot create %s (%d).\n", path, errno); - close(*fd); + strncpy(sunx.sun_path, (char*)pLstn->sockName, sizeof(sunx.sun_path)); + pLstn->fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if(pLstn->fd < 0 || bind(pLstn->fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || + chmod((char*)pLstn->sockName, 0666) < 0) { + errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", pLstn->sockName); + dbgprintf("cannot create %s (%d).\n", pLstn->sockName, errno); + close(pLstn->fd); + pLstn->fd = -1; ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } finalize_it: @@ -230,16 +232,15 @@ finalize_it: static inline rsRetVal -openLogSocket(const char *path, int bCreatePath, int *pfd) +openLogSocket(lstn_t *pLstn) { DEFiRet; - int fd; int one; - if (path[0] == '\0') + if(pLstn->sockName[0] == '\0') return -1; - if (strcmp(path, _PATH_LOG) == 0) { + if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { int r; /* System log socket code. Check whether an FD was passed in from systemd. If @@ -257,8 +258,8 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) } if (r == 1) { - fd = SD_LISTEN_FDS_START; - r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); + pLstn->fd = SD_LISTEN_FDS_START; + r = sd_is_socket_unix(pLstn->fd, SOCK_DGRAM, -1, _PATH_LOG, 0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); @@ -269,42 +270,39 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } } else { - CHKiRet(createLogSocket(path, bCreatePath, &fd)); + CHKiRet(createLogSocket(pLstn)); } } else { - CHKiRet(createLogSocket(path, bCreatePath, &fd)); + CHKiRet(createLogSocket(pLstn)); } - one = 1; - if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + if(pLstn->bUseCreds) { + one = 1; + if(setsockopt(pLstn->fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_PASSCRED failed on '%s'", pLstn->sockName); + pLstn->bUseCreds = 0; + } + if(setsockopt(pLstn->fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS failed on '%s'", pLstn->sockName); + pLstn->bUseCreds = 0; + } } + +#if 0 + // SO_TIMESTAMP currently does not work for an unknown reason. Any help is appreciated! one = 1; - if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); - close(fd); + if(setsockopt(pLstn->fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", pLstn->sockName); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } +#endif - - -static socklen_t two =sizeof(one); - if(getsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one,&two) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); +finalize_it: + if(iRet != RS_RET_OK) { + close(pLstn->fd); + pLstn->fd = -1; } -dbgprintf("imuxsock: SO_TIMESTAMP is %d\n", one); - *pfd = fd; -finalize_it: RETiRet; } @@ -321,7 +319,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) if(cred == NULL) return; - lenPID = snprintf(bufPID, sizeof(bufPID), "[%u]:", (unsigned) cred->pid); + lenPID = snprintf(bufPID, sizeof(bufPID), "[%lu]:", (unsigned long) cred->pid); for(i = *lenTag ; i >= 0 && bufTAG[i] != '[' ; --i) /*JUST SKIP*/; @@ -342,7 +340,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) * can also mangle it if necessary. */ static inline rsRetVal -SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) +SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) { msg_t *pMsg; int lenMsg; @@ -356,7 +354,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) CHKiRet(msgConstruct(&pMsg)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); - MsgSetFlowControlType(pMsg, listeners[iSock].flowCtl); + MsgSetFlowControlType(pMsg, pLstn->flowCtl); // TODO: handle format errors parse = pRcv; @@ -372,17 +370,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); -dbgprintf("imuxsock: submit: facil %d, sever %d\n", pMsg->iFacility, pMsg->iSeverity); parse++; lenMsg--; /* '>' */ -dbgprintf("imuxsock: submit: stage 2\n"); if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { dbgprintf("we have a problem, invalid timestamp in msg!\n"); } /* pull tag */ -dbgprintf("imuxsock: submit: stage 3\n"); i = 0; while(lenMsg > 0 && *parse != ' ' && i < CONF_TAG_MAXSIZE) { @@ -395,19 +390,13 @@ dbgprintf("imuxsock: submit: stage 3\n"); MsgSetMSGoffs(pMsg, lenRcv - lenMsg); - // TODO: here we need to mangle the raw message if we need to - // "fix up" the user pid. In the long term, it may make more sense - // to add this (somehow) to the message object (problem: at this stage, - // it is not fully available, eg no structured data). Alternatively, we - // may use a custom parser for our messages, but that doesn't play too well - // with the rest of the system. - if(listeners[iSock].bParseHost) { - pMsg->msgFlags = listeners[iSock].flags | PARSE_HOSTNAME; + if(pLstn->bParseHost) { + pMsg->msgFlags = pLstn->flags | PARSE_HOSTNAME; } else { - pMsg->msgFlags = listeners[iSock].flags; + pMsg->msgFlags = pLstn->flags; } - MsgSetRcvFrom(pMsg, listeners[iSock].hostName); + MsgSetRcvFrom(pMsg, pLstn->hostName); CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); @@ -424,21 +413,20 @@ finalize_it: * of the socket which is to be processed. This eases access to the * growing number of properties. -- rgerhards, 2008-08-01 */ -static rsRetVal readSocket(int fd, int iSock) +static rsRetVal readSocket(lstn_t *pLstn) { DEFiRet; int iRcvd; int iMaxLine; struct msghdr msgh; struct iovec msgiov; - static int ratelimitErrmsg = 0; // TODO: atomic OPS struct cmsghdr *cm; struct ucred *cred; uchar bufRcv[4096+1]; - char aux[1024]; + char aux[128]; uchar *pRcv = NULL; /* receive buffer */ - assert(iSock >= 0); + assert(pLstn->fd >= 0); iMaxLine = glbl.GetMaxLine(); @@ -456,38 +444,34 @@ static rsRetVal readSocket(int fd, int iSock) memset(&msgh, 0, sizeof(msgh)); memset(&msgiov, 0, sizeof(msgiov)); - memset(&aux, 0, sizeof(aux)); + if(pLstn->bUseCreds) { + memset(&aux, 0, sizeof(aux)); + msgh.msg_control = aux; + msgh.msg_controllen = sizeof(aux); + } msgiov.iov_base = pRcv; msgiov.iov_len = iMaxLine; msgh.msg_iov = &msgiov; msgh.msg_iovlen = 1; - msgh.msg_control = aux; - msgh.msg_controllen = sizeof(aux); - iRcvd = recvmsg(fd, &msgh, MSG_DONTWAIT); -/* iRcvd = recv(fd, pRcv, iMaxLine, 0); - */ + iRcvd = recvmsg(pLstn->fd, &msgh, MSG_DONTWAIT); - dbgprintf("Message from UNIX socket: #%d\n", fd); - if (iRcvd > 0) { - ratelimitErrmsg = 0; - - -dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); - cred = NULL; - for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { -dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); - if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { - cred = (struct ucred*) CMSG_DATA(cm); - dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); - //break; - } - } -dbgprintf("XXX: post CM loop\n"); - - - - CHKiRet(SubmitMsg(pRcv, iRcvd, iSock, cred)); - } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { + dbgprintf("Message from UNIX socket: #%d\n", pLstn->fd); + if(iRcvd > 0) { + cred = NULL; + if(pLstn->bUseCreds) { + dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); + for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { + dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { + cred = (struct ucred*) CMSG_DATA(cm); + dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); + //break; + } + } + dbgprintf("XXX: post CM loop\n"); + } + CHKiRet(SubmitMsg(pRcv, iRcvd, pLstn, cred)); + } else if(iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("UNIX socket error: %d = %s.\n", errno, errStr); @@ -555,7 +539,7 @@ CODESTARTrunInput if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ if ((fd = listeners[i].fd) != -1 && FD_ISSET(fd, pReadfds)) { - readSocket(fd, i); + readSocket(&(listeners[i])); --nfds; /* indicate we have processed one */ } } @@ -585,12 +569,12 @@ CODESTARTwillRun # endif if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; + listeners[0].bUseCreds = bUseCredsSysSock; /* initialize and return if will run or not */ actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfd ; i++) { - if(openLogSocket((char*) listeners[i].sockName, listeners[i].bCreateSockPath, - &(listeners[i].fd)) == RS_RET_OK) { + if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } @@ -682,7 +666,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bIgnoreTimestamp = 1; bUseFlowCtl = 0; bUseCreds = 0; - bCreateSockPath = DFLT_bCreateSockPath; + bUseCredsSysSock = 0; + bCreatePath = DFLT_bCreatePath; return RS_RET_OK; } @@ -709,7 +694,7 @@ CODEmodInit_QueryRegCFSLineHdlr listeners[0].fd = -1; listeners[0].bParseHost = 0; listeners[0].bUseCreds = 0; - listeners[0].bCreateSockPath = 0; + listeners[0].bCreatePath = 0; /* initialize socket names */ for(i = 1 ; i < MAXFUNIX ; ++i) { @@ -738,9 +723,11 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketflowcontrol", 0, eCmdHdlrBinary, NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, - NULL, &bCreateSockPath, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusespidfromsystem", 0, eCmdHdlrBinary, + NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit v1.2.3 From b5da6352830e9841dd367b8490d79461adb5cb22 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 15:06:30 +0200 Subject: first shot at imuxsock ratelimiting works, but at a global level, need to go down to pid or cgroup --- plugins/imuxsock/imuxsock.c | 98 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index df2b8efc..ccd31b44 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -81,6 +81,15 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +STATSCOUNTER_DEF(ctrLostRatelimit, mutCtrLostRatelimit) + +struct rs_ratelimit_state { + unsigned short interval; + unsigned short burst; + unsigned done; + unsigned missed; + time_t begin; +} ratelimiter; /* structure to describe a specific listener */ typedef struct lstn_s { @@ -113,6 +122,70 @@ static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TA static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ +#define DFLT_ratelimitInterval 2 +static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ +#define DFLT_ratelimitBurst 200 +static int ratelimitBurst = DFLT_ratelimitBurst; /* max nbr of messages in interval */ + + + +static void +initRatelimitState(struct rs_ratelimit_state *rs, unsigned short interval, unsigned short burst) +{ + rs->interval = interval; + rs->burst = burst; + rs->done = 0; + rs->missed = 0; + rs->begin = 0; +} + + +/* ratelimiting support, modelled after the linux kernel + * returns 1 if message is within rate limit and shall be + * processed, 0 otherwise. + * This implementation is NOT THREAD-SAFE and must not + * be called concurrently. + */ +static inline int +withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) +{ + int ret; + uchar msgbuf[1024]; + + if(rs->interval == 0) { + ret = 1; + goto finalize_it; + } + + assert(rs->burst != 0); + + if(rs->begin == 0) + rs->begin = tt; + + /* resume if we go out of out time window */ + if(tt > rs->begin + rs->interval) { + if(rs->missed) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "imuxsock lost %u messages due to rate-limiting", rs->missed); + logmsgInternal(0, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + rs->missed = 0; + } + rs->begin = 0; + rs->done = 0; + } + + /* do actual limit check */ + if(rs->burst > rs->done) { + rs->done++; + ret = 1; + } else { + rs->missed++; + ret = 0; + } + +finalize_it: + return ret; +} /* set the timestamp ignore / not ignore option for the system @@ -348,14 +421,23 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) uchar *parse; int pri; uchar bufParseTAG[CONF_TAG_MAXSIZE]; + struct syslogTime st; + time_t tt; DEFiRet; + datetime.getCurrTime(&st, &tt); + if(!withinRatelimit(&ratelimiter, tt)) { + STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); + FINALIZE; + } + /* we now create our own message object and submit it to the queue */ - CHKiRet(msgConstruct(&pMsg)); + CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); + // TODO: handle format errors parse = pRcv; lenMsg = lenRcv; @@ -504,6 +586,8 @@ CODESTARTrunInput * signalled to do so. This, however, is handled by the framework, * right into the sleep below. */ + initRatelimitState(&ratelimiter, ratelimitInterval, ratelimitBurst); + while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -668,6 +752,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bUseCreds = 0; bUseCredsSysSock = 0; bCreatePath = DFLT_bCreatePath; + ratelimitInterval = DFLT_ratelimitInterval; + ratelimitBurst = DFLT_ratelimitBurst; return RS_RET_OK; } @@ -726,10 +812,12 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitinterval", 0, eCmdHdlrInt, + NULL, &ratelimitInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitburst", 0, eCmdHdlrInt, + NULL, &ratelimitBurst, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); /* the following one is a (dirty) trick: the system log socket is not added via @@ -742,12 +830,16 @@ CODEmodInit_QueryRegCFSLineHdlr setSystemLogTimestampIgnore, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); /* support statistics gathering */ CHKiRet(statsobj.Construct(&modStats)); CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), ctrType_IntCtr, &ctrSubmit)); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("lost.ratelimit"), + ctrType_IntCtr, &ctrLostRatelimit)); CHKiRet(statsobj.ConstructFinalize(modStats)); ENDmodInit -- cgit v1.2.3 From d748c68c09ae7d4f8c22c4245e5509019c59511c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 11:32:48 +0200 Subject: added some generic hashtable code by Christopher Clark found at http://www.cl.cam.ac.uk/~cwc22/hashtable/ --- plugins/imuxsock/Makefile.am | 2 +- plugins/imuxsock/imuxsock.c | 1 + runtime/Makefile.am | 2 + runtime/hashtable/Makefile | 26 ++++ runtime/hashtable/hashtable.c | 275 ++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.h | 199 ++++++++++++++++++++++++ runtime/hashtable/hashtable_itr.c | 188 +++++++++++++++++++++++ runtime/hashtable/hashtable_itr.h | 112 ++++++++++++++ runtime/hashtable/hashtable_private.h | 85 +++++++++++ runtime/hashtable/hashtable_utility.c | 71 +++++++++ runtime/hashtable/hashtable_utility.h | 55 +++++++ runtime/hashtable/tester.c | 270 +++++++++++++++++++++++++++++++++ 12 files changed, 1285 insertions(+), 1 deletion(-) create mode 100644 runtime/hashtable/Makefile create mode 100644 runtime/hashtable/hashtable.c create mode 100644 runtime/hashtable/hashtable.h create mode 100644 runtime/hashtable/hashtable_itr.c create mode 100644 runtime/hashtable/hashtable_itr.h create mode 100644 runtime/hashtable/hashtable_private.h create mode 100644 runtime/hashtable/hashtable_utility.c create mode 100644 runtime/hashtable/hashtable_utility.h create mode 100644 runtime/hashtable/tester.c diff --git a/plugins/imuxsock/Makefile.am b/plugins/imuxsock/Makefile.am index 28f9f9e3..34a0ad9a 100644 --- a/plugins/imuxsock/Makefile.am +++ b/plugins/imuxsock/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imuxsock.la imuxsock_la_SOURCES = imuxsock.c -imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I../../runtime/hashtable -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imuxsock_la_LDFLAGS = -module -avoid-version imuxsock_la_LIBADD = $(RSRT_LIBS) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index ccd31b44..b53bb379 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -52,6 +52,7 @@ #include "sd-daemon.h" #include "statsobj.h" #include "datetime.h" +#include "hashtable.h" MODULE_TYPE_INPUT diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 01b224e7..d657b6ca 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -95,6 +95,8 @@ librsyslog_la_SOURCES = \ ../parse.c \ ../parse.h \ \ + hashtable/hashtable.c \ + \ ../outchannel.c \ ../outchannel.h \ ../template.c \ diff --git a/runtime/hashtable/Makefile b/runtime/hashtable/Makefile new file mode 100644 index 00000000..3b7b5e9f --- /dev/null +++ b/runtime/hashtable/Makefile @@ -0,0 +1,26 @@ + +tester: hashtable.o tester.o hashtable_itr.o + gcc -g -Wall -O -lm -o tester hashtable.o hashtable_itr.o tester.o + +all: tester old_tester + +tester.o: tester.c + gcc -g -Wall -O -c tester.c -o tester.o + +old_tester: hashtable_powers.o tester.o hashtable_itr.o + gcc -g -Wall -O -o old_tester hashtable_powers.o hashtable_itr.o tester.o + +hashtable_powers.o: hashtable_powers.c + gcc -g -Wall -O -c hashtable_powers.c -o hashtable_powers.o + +hashtable.o: hashtable.c + gcc -g -Wall -O -c hashtable.c -o hashtable.o + +hashtable_itr.o: hashtable_itr.c + gcc -g -Wall -O -c hashtable_itr.c -o hashtable_itr.o + +tidy: + rm *.o + +clean: tidy + rm -f tester old_tester diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c new file mode 100644 index 00000000..a10e3bc6 --- /dev/null +++ b/runtime/hashtable/hashtable.c @@ -0,0 +1,275 @@ +/* Copyright (C) 2004 Christopher Clark */ +/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include +#include +#include +#include + +/* +Credit for primes table: Aaron Krowne + http://br.endernet.org/~akrowne/ + http://planetmath.org/encyclopedia/GoodHashTablePrimes.html +*/ +static const unsigned int primes[] = { +53, 97, 193, 389, +769, 1543, 3079, 6151, +12289, 24593, 49157, 98317, +196613, 393241, 786433, 1572869, +3145739, 6291469, 12582917, 25165843, +50331653, 100663319, 201326611, 402653189, +805306457, 1610612741 +}; +const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); +const float max_load_factor = 0.65; + +/*****************************************************************************/ +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashf) (void*), + int (*eqf) (void*,void*)) +{ + struct hashtable *h; + unsigned int pindex, size = primes[0]; + /* Check requested hashtable isn't too large */ + if (minsize > (1u << 30)) return NULL; + /* Enforce size as prime */ + for (pindex=0; pindex < prime_table_length; pindex++) { + if (primes[pindex] > minsize) { size = primes[pindex]; break; } + } + h = (struct hashtable *)malloc(sizeof(struct hashtable)); + if (NULL == h) return NULL; /*oom*/ + h->table = (struct entry **)malloc(sizeof(struct entry*) * size); + if (NULL == h->table) { free(h); return NULL; } /*oom*/ + memset(h->table, 0, size * sizeof(struct entry *)); + h->tablelength = size; + h->primeindex = pindex; + h->entrycount = 0; + h->hashfn = hashf; + h->eqfn = eqf; + h->loadlimit = (unsigned int) ceil(size * max_load_factor); + return h; +} + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k) +{ + /* Aim to protect against poor hash functions by adding logic here + * - logic taken from java 1.4 hashtable source */ + unsigned int i = h->hashfn(k); + i += ~(i << 9); + i ^= ((i >> 14) | (i << 18)); /* >>> */ + i += (i << 4); + i ^= ((i >> 10) | (i << 22)); /* >>> */ + return i; +} + +/*****************************************************************************/ +static int +hashtable_expand(struct hashtable *h) +{ + /* Double the size of the table to accomodate more entries */ + struct entry **newtable; + struct entry *e; + struct entry **pE; + unsigned int newsize, i, idx; + /* Check we're not hitting max capacity */ + if (h->primeindex == (prime_table_length - 1)) return 0; + newsize = primes[++(h->primeindex)]; + + newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); + if (NULL != newtable) + { + memset(newtable, 0, newsize * sizeof(struct entry *)); + /* This algorithm is not 'stable'. ie. it reverses the list + * when it transfers entries between the tables */ + for (i = 0; i < h->tablelength; i++) { + while (NULL != (e = h->table[i])) { + h->table[i] = e->next; + idx = indexFor(newsize,e->h); + e->next = newtable[idx]; + newtable[idx] = e; + } + } + free(h->table); + h->table = newtable; + } + /* Plan B: realloc instead */ + else + { + newtable = (struct entry **) + realloc(h->table, newsize * sizeof(struct entry *)); + if (NULL == newtable) { (h->primeindex)--; return 0; } + h->table = newtable; + memset(newtable[h->tablelength], 0, newsize - h->tablelength); + for (i = 0; i < h->tablelength; i++) { + for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { + idx = indexFor(newsize,e->h); + if (idx == i) + { + pE = &(e->next); + } + else + { + *pE = e->next; + e->next = newtable[idx]; + newtable[idx] = e; + } + } + } + } + h->tablelength = newsize; + h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + return -1; +} + +/*****************************************************************************/ +unsigned int +hashtable_count(struct hashtable *h) +{ + return h->entrycount; +} + +/*****************************************************************************/ +int +hashtable_insert(struct hashtable *h, void *k, void *v) +{ + /* This method allows duplicate keys - but they shouldn't be used */ + unsigned int idx; + struct entry *e; + if (++(h->entrycount) > h->loadlimit) + { + /* Ignore the return value. If expand fails, we should + * still try cramming just this value into the existing table + * -- we may not have memory for a larger table, but one more + * element may be ok. Next time we insert, we'll try expanding again.*/ + hashtable_expand(h); + } + e = (struct entry *)malloc(sizeof(struct entry)); + if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ + e->h = hash(h,k); + idx = indexFor(h->tablelength,e->h); + e->k = k; + e->v = v; + e->next = h->table[idx]; + h->table[idx] = e; + return -1; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_search(struct hashtable *h, void *k) +{ + struct entry *e; + unsigned int hashvalue, idx; + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hashvalue); + e = h->table[idx]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_remove(struct hashtable *h, void *k) +{ + /* TODO: consider compacting the table when the load factor drops enough, + * or provide a 'compact' method. */ + + struct entry *e; + struct entry **pE; + void *v; + unsigned int hashvalue, idx; + + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hash(h,k)); + pE = &(h->table[idx]); + e = *pE; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + *pE = e->next; + h->entrycount--; + v = e->v; + freekey(e->k); + free(e); + return v; + } + pE = &(e->next); + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +/* destroy */ +void +hashtable_destroy(struct hashtable *h, int free_values) +{ + unsigned int i; + struct entry *e, *f; + struct entry **table = h->table; + if (free_values) + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } + } + } + else + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f); } + } + } + free(h->table); + free(h); +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.h b/runtime/hashtable/hashtable.h new file mode 100644 index 00000000..b90781ab --- /dev/null +++ b/runtime/hashtable/hashtable.h @@ -0,0 +1,199 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_H__ +#define __HASHTABLE_CWC22_H__ + +struct hashtable; + +/* Example of use: + * + * struct hashtable *h; + * struct some_key *k; + * struct some_value *v; + * + * static unsigned int hash_from_key_fn( void *k ); + * static int keys_equal_fn ( void *key1, void *key2 ); + * + * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); + * k = (struct some_key *) malloc(sizeof(struct some_key)); + * v = (struct some_value *) malloc(sizeof(struct some_value)); + * + * (initialise k and v to suitable values) + * + * if (! hashtable_insert(h,k,v) ) + * { exit(-1); } + * + * if (NULL == (found = hashtable_search(h,k) )) + * { printf("not found!"); } + * + * if (NULL == (found = hashtable_remove(h,k) )) + * { printf("Not found\n"); } + * + */ + +/* Macros may be used to define type-safe(r) hashtable access functions, with + * methods specialized to take known key and value types as parameters. + * + * Example: + * + * Insert this at the start of your file: + * + * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); + * + * This defines the functions 'insert_some', 'search_some' and 'remove_some'. + * These operate just like hashtable_insert etc., with the same parameters, + * but their function signatures have 'struct some_key *' rather than + * 'void *', and hence can generate compile time errors if your program is + * supplying incorrect data as a key (and similarly for value). + * + * Note that the hash and key equality functions passed to create_hashtable + * still take 'void *' parameters instead of 'some key *'. This shouldn't be + * a difficult issue as they're only defined and passed once, and the other + * functions will ensure that only valid keys are supplied to them. + * + * The cost for this checking is increased code size and runtime overhead + * - if performance is important, it may be worth switching back to the + * unsafe methods once your program has been debugged with the safe methods. + * This just requires switching to some simple alternative defines - eg: + * #define insert_some hashtable_insert + * + */ + +/***************************************************************************** + * create_hashtable + + * @name create_hashtable + * @param minsize minimum initial size of hashtable + * @param hashfunction function for hashing keys + * @param key_eq_fn function for determining key equality + * @return newly created hashtable or NULL on failure + */ + +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashfunction) (void*), + int (*key_eq_fn) (void*,void*)); + +/***************************************************************************** + * hashtable_insert + + * @name hashtable_insert + * @param h the hashtable to insert into + * @param k the key - hashtable claims ownership and will free on removal + * @param v the value - does not claim ownership + * @return non-zero for successful insertion + * + * This function will cause the table to expand if the insertion would take + * the ratio of entries to table size over the maximum load factor. + * + * This function does not check for repeated insertions with a duplicate key. + * The value returned when using a duplicate key is undefined -- when + * the hashtable changes size, the order of retrieval of duplicate key + * entries is reversed. + * If in doubt, remove before insert. + */ + +int +hashtable_insert(struct hashtable *h, void *k, void *v); + +#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ +int fnname (struct hashtable *h, keytype *k, valuetype *v) \ +{ \ + return hashtable_insert(h,k,v); \ +} + +/***************************************************************************** + * hashtable_search + + * @name hashtable_search + * @param h the hashtable to search + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * +hashtable_search(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_search(h,k)); \ +} + +/***************************************************************************** + * hashtable_remove + + * @name hashtable_remove + * @param h the hashtable to remove the item from + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * /* returns value */ +hashtable_remove(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_remove(h,k)); \ +} + + +/***************************************************************************** + * hashtable_count + + * @name hashtable_count + * @param h the hashtable + * @return the number of items stored in the hashtable + */ +unsigned int +hashtable_count(struct hashtable *h); + + +/***************************************************************************** + * hashtable_destroy + + * @name hashtable_destroy + * @param h the hashtable + * @param free_values whether to call 'free' on the remaining values + */ + +void +hashtable_destroy(struct hashtable *h, int free_values); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_itr.c b/runtime/hashtable/hashtable_itr.c new file mode 100644 index 00000000..5dced841 --- /dev/null +++ b/runtime/hashtable/hashtable_itr.c @@ -0,0 +1,188 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_itr.h" +#include /* defines NULL */ + +/*****************************************************************************/ +/* hashtable_iterator - iterator constructor */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h) +{ + unsigned int i, tablelength; + struct hashtable_itr *itr = (struct hashtable_itr *) + malloc(sizeof(struct hashtable_itr)); + if (NULL == itr) return NULL; + itr->h = h; + itr->e = NULL; + itr->parent = NULL; + tablelength = h->tablelength; + itr->index = tablelength; + if (0 == h->entrycount) return itr; + + for (i = 0; i < tablelength; i++) + { + if (NULL != h->table[i]) + { + itr->e = h->table[i]; + itr->index = i; + break; + } + } + return itr; +} + +/*****************************************************************************/ +/* key - return the key of the (key,value) pair at the current position */ +/* value - return the value of the (key,value) pair at the current position */ + +void * +hashtable_iterator_key(struct hashtable_itr *i) +{ return i->e->k; } + +void * +hashtable_iterator_value(struct hashtable_itr *i) +{ return i->e->v; } + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr) +{ + unsigned int j,tablelength; + struct entry **table; + struct entry *next; + if (NULL == itr->e) return 0; /* stupidity check */ + + next = itr->e->next; + if (NULL != next) + { + itr->parent = itr->e; + itr->e = next; + return -1; + } + tablelength = itr->h->tablelength; + itr->parent = NULL; + if (tablelength <= (j = ++(itr->index))) + { + itr->e = NULL; + return 0; + } + table = itr->h->table; + while (NULL == (next = table[j])) + { + if (++j >= tablelength) + { + itr->index = tablelength; + itr->e = NULL; + return 0; + } + } + itr->index = j; + itr->e = next; + return -1; +} + +/*****************************************************************************/ +/* remove - remove the entry at the current iterator position + * and advance the iterator, if there is a successive + * element. + * If you want the value, read it before you remove: + * beware memory leaks if you don't. + * Returns zero if end of iteration. */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr) +{ + struct entry *remember_e, *remember_parent; + int ret; + + /* Do the removal */ + if (NULL == (itr->parent)) + { + /* element is head of a chain */ + itr->h->table[itr->index] = itr->e->next; + } else { + /* element is mid-chain */ + itr->parent->next = itr->e->next; + } + /* itr->e is now outside the hashtable */ + remember_e = itr->e; + itr->h->entrycount--; + freekey(remember_e->k); + + /* Advance the iterator, correcting the parent */ + remember_parent = itr->parent; + ret = hashtable_iterator_advance(itr); + if (itr->parent == remember_e) { itr->parent = remember_parent; } + free(remember_e); + return ret; +} + +/*****************************************************************************/ +int /* returns zero if not found */ +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k) +{ + struct entry *e, *parent; + unsigned int hashvalue, index; + + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + + e = h->table[index]; + parent = NULL; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + itr->index = index; + itr->e = e; + itr->parent = parent; + itr->h = h; + return -1; + } + parent = e; + e = e->next; + } + return 0; +} + + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_itr.h b/runtime/hashtable/hashtable_itr.h new file mode 100644 index 00000000..eea699a7 --- /dev/null +++ b/runtime/hashtable/hashtable_itr.h @@ -0,0 +1,112 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_ITR_CWC22__ +#define __HASHTABLE_ITR_CWC22__ +#include "hashtable.h" +#include "hashtable_private.h" /* needed to enable inlining */ + +/*****************************************************************************/ +/* This struct is only concrete here to allow the inlining of two of the + * accessor functions. */ +struct hashtable_itr +{ + struct hashtable *h; + struct entry *e; + struct entry *parent; + unsigned int index; +}; + + +/*****************************************************************************/ +/* hashtable_iterator + */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h); + +/*****************************************************************************/ +/* hashtable_iterator_key + * - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_key(struct hashtable_itr *i) +{ + return i->e->k; +} + +/*****************************************************************************/ +/* value - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_value(struct hashtable_itr *i) +{ + return i->e->v; +} + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* remove - remove current element and advance the iterator to the next element + * NB: if you need the value to free it, read it before + * removing. ie: beware memory leaks! + * returns zero if advanced to end of table */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* search - overwrite the supplied iterator, to point to the entry + * matching the supplied key. + h points to the hashtable to be searched. + * returns zero if not found. */ +int +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ +int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ +{ \ + return (hashtable_iterator_search(i,h,k)); \ +} + + + +#endif /* __HASHTABLE_ITR_CWC22__*/ + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h new file mode 100644 index 00000000..3e95f600 --- /dev/null +++ b/runtime/hashtable/hashtable_private.h @@ -0,0 +1,85 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_PRIVATE_CWC22_H__ +#define __HASHTABLE_PRIVATE_CWC22_H__ + +#include "hashtable.h" + +/*****************************************************************************/ +struct entry +{ + void *k, *v; + unsigned int h; + struct entry *next; +}; + +struct hashtable { + unsigned int tablelength; + struct entry **table; + unsigned int entrycount; + unsigned int loadlimit; + unsigned int primeindex; + unsigned int (*hashfn) (void *k); + int (*eqfn) (void *k1, void *k2); +}; + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k); + +/*****************************************************************************/ +/* indexFor */ +static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) { + return (hashvalue % tablelength); +}; + +/* Only works if tablelength == 2^N */ +/*static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) +{ + return (hashvalue & (tablelength - 1u)); +} +*/ + +/*****************************************************************************/ +#define freekey(X) free(X) +/*define freekey(X) ; */ + + +/*****************************************************************************/ + +#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_utility.c b/runtime/hashtable/hashtable_utility.c new file mode 100644 index 00000000..c3176709 --- /dev/null +++ b/runtime/hashtable/hashtable_utility.c @@ -0,0 +1,71 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_utility.h" +#include +#include +#include + +/*****************************************************************************/ +/* hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * */ +int +hashtable_change(struct hashtable *h, void *k, void *v) +{ + struct entry *e; + unsigned int hashvalue, index; + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + e = h->table[index]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + free(e->v); + e->v = v; + return -1; + } + e = e->next; + } + return 0; +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_utility.h b/runtime/hashtable/hashtable_utility.h new file mode 100644 index 00000000..56a0ffd1 --- /dev/null +++ b/runtime/hashtable/hashtable_utility.h @@ -0,0 +1,55 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_UTILITY_H__ +#define __HASHTABLE_CWC22_UTILITY_H__ + +/***************************************************************************** + * hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * @name hashtable_change + * @param h the hashtable + * @param key + * @param value + * + */ +int +hashtable_change(struct hashtable *h, void *k, void *v); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/tester.c b/runtime/hashtable/tester.c new file mode 100644 index 00000000..4678ffa8 --- /dev/null +++ b/runtime/hashtable/tester.c @@ -0,0 +1,270 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_itr.h" +#include +#include +#include /* for memcmp */ + +static const int ITEM_COUNT = 4000; + +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; + +/*****************************************************************************/ +struct key +{ + uint32_t one_ip; uint32_t two_ip; uint16_t one_port; uint16_t two_port; +}; + +struct value +{ + char *id; +}; + +DEFINE_HASHTABLE_INSERT(insert_some, struct key, struct value); +DEFINE_HASHTABLE_SEARCH(search_some, struct key, struct value); +DEFINE_HASHTABLE_REMOVE(remove_some, struct key, struct value); +DEFINE_HASHTABLE_ITERATOR_SEARCH(search_itr_some, struct key); + + +/*****************************************************************************/ +static unsigned int +hashfromkey(void *ky) +{ + struct key *k = (struct key *)ky; + return (((k->one_ip << 17) | (k->one_ip >> 15)) ^ k->two_ip) + + (k->one_port * 17) + (k->two_port * 13 * 29); +} + +static int +equalkeys(void *k1, void *k2) +{ + return (0 == memcmp(k1,k2,sizeof(struct key))); +} + +/*****************************************************************************/ +int +main(int argc, char **argv) +{ + struct key *k, *kk; + struct value *v, *found; + struct hashtable *h; + struct hashtable_itr *itr; + int i; + + h = create_hashtable(16, hashfromkey, equalkeys); + if (NULL == h) exit(-1); /*oom*/ + + +/*****************************************************************************/ +/* Insertion */ + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) exit(-1); /*oom*/ + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable search */ + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = search_some(h,k))) { + printf("BUG: key not found\n"); + } + } + +/*****************************************************************************/ +/* Hashtable iteration */ + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(h); + i = 0; + if (hashtable_count(h) > 0) + { + do { + kk = hashtable_iterator_key(itr); + v = hashtable_iterator_value(itr); + /* here (kk,v) are a valid (key, value) pair */ + /* We could call 'hashtable_remove(h,kk)' - and this operation + * 'free's kk. However, the iterator is then broken. + * This is why hashtable_iterator_remove exists - see below. + */ + i++; + + } while (hashtable_iterator_advance(itr)); + } + printf("Iterated through %u entries.\n", i); + +/*****************************************************************************/ +/* Hashtable iterator search */ + + /* Try the search some method */ + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr,h,k)) { + printf("BUG: key not found searching with iterator"); + } + } + +/*****************************************************************************/ +/* Hashtable removal */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = remove_some(h,k))) { + printf("BUG: key not found for removal\n"); + } + } + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy and create */ + + hashtable_destroy(h, 1); + h = NULL; + free(k); + + h = create_hashtable(160, hashfromkey, equalkeys); + if (NULL == h) { + printf("out of memory allocating second hashtable\n"); + return 1; + } + +/*****************************************************************************/ +/* Hashtable insertion */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) + { + printf("out of memory inserting into second hashtable\n"); + return 1; + } + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable iterator search and iterator remove */ + + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = ITEM_COUNT - 1; i >= 0; i = i - 7) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr, h, k)) { + printf("BUG: key %u not found for search preremoval using iterator\n", i); + return 1; + } + if (0 == hashtable_iterator_remove(itr)) { + printf("BUG: key not found for removal using iterator\n"); + return 1; + } + } + free(itr); + +/*****************************************************************************/ +/* Hashtable iterator remove and advance */ + + for (itr = hashtable_iterator(h); + hashtable_iterator_remove(itr) != 0; ) { + ; + } + free(itr); + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy */ + + hashtable_destroy(h, 1); + free(k); + return 0; +} + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ -- cgit v1.2.3 From 01a8807174d91a7936345d1172a87f98bbba61c4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 12:13:19 +0200 Subject: imuxsock: changed to per-pid ratelimiting --- plugins/imuxsock/imuxsock.c | 83 +++++++++++++++++++++++++++++++++++++++++---- runtime/hashtable/README | 11 ++++++ tools/Makefile.am | 2 +- 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 runtime/hashtable/README diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index b53bb379..728456b6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-20 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -83,6 +83,7 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) STATSCOUNTER_DEF(ctrLostRatelimit, mutCtrLostRatelimit) +STATSCOUNTER_DEF(ctrNumRatelimiters, mutCtrNumRatelimiters) struct rs_ratelimit_state { unsigned short interval; @@ -90,7 +91,30 @@ struct rs_ratelimit_state { unsigned done; unsigned missed; time_t begin; -} ratelimiter; +}; +typedef struct rs_ratelimit_state rs_ratelimit_state_t; + +/* support for per-process ratelimiting */ +static struct hashtable *hashtab; + +/* a very simple "hash function" for process IDs - we simply use the + * pid itself: it is quite expected that all pids may log some time, but + * from a collision point of view it is likely that long-running daemons + * start early and so will stay right in the top spots of the + * collision list. + */ +static unsigned int +hash_from_key_fn(void *k) +{ + return((unsigned) *((pid_t*) k)); +} + +static int +key_equals_fn(void *key1, void *key2) +{ + return *((pid_t*) key1) == *((pid_t*) key2); +} + /* structure to describe a specific listener */ typedef struct lstn_s { @@ -381,6 +405,43 @@ finalize_it: } +/* find ratelimiter to use for this message. Currently, we use the + * pid, but may change to cgroup later (probably via a config switch). + * Returns NULL if not found. + */ +static inline rsRetVal +findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) +{ + rs_ratelimit_state_t *rl; + int r; + pid_t *keybuf; + DEFiRet; + + if(cred == NULL) + FINALIZE; + + rl = hashtable_search(hashtab, &cred->pid); + if(rl == NULL) { + /* we need to add a new ratelimiter, process not seen before! */ + dbgprintf("imuxsock: no ratelimiter for pid %lu, creating one\n", + (unsigned long) cred->pid); + STATSCOUNTER_INC(ctrNumRatelimiters, mutCtrNumRatelimiters); + CHKmalloc(rl = malloc(sizeof(rs_ratelimit_state_t))); + CHKmalloc(keybuf = malloc(sizeof(pid_t))); + *keybuf = cred->pid; + initRatelimitState(rl, ratelimitInterval, ratelimitBurst); + r = hashtable_insert(hashtab, keybuf, rl); + if(r == 0) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + + *prl = rl; + +finalize_it: + RETiRet; +} + + /* patch correct pid into tag. bufTAG MUST be CONF_TAG_MAXSIZE long! */ static inline void @@ -424,10 +485,13 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) uchar bufParseTAG[CONF_TAG_MAXSIZE]; struct syslogTime st; time_t tt; + rs_ratelimit_state_t *ratelimiter; DEFiRet; + findRatelimiter(cred, &ratelimiter); /* ignore error, better so than others... */ + datetime.getCurrTime(&st, &tt); - if(!withinRatelimit(&ratelimiter, tt)) { + if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); FINALIZE; } @@ -587,8 +651,6 @@ CODESTARTrunInput * signalled to do so. This, however, is handled by the framework, * right into the sleep below. */ - initRatelimitState(&ratelimiter, ratelimitInterval, ratelimitBurst); - while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -675,6 +737,8 @@ CODESTARTwillRun CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); + CHKmalloc(hashtab = create_hashtable(1000, hash_from_key_fn, key_equals_fn)); + finalize_it: ENDwillRun @@ -706,6 +770,11 @@ CODESTARTafterRun if(pInputName != NULL) prop.Destruct(&pInputName); + + if(hashtab != NULL) { + hashtable_destroy(hashtab, 1); /* 1 => free all values automatically */ + hashtab = NULL; + } ENDafterRun @@ -839,8 +908,10 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), ctrType_IntCtr, &ctrSubmit)); - CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("lost.ratelimit"), + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("ratelimit.discarded"), ctrType_IntCtr, &ctrLostRatelimit)); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("ratelimit.numratelimiters"), + ctrType_IntCtr, &ctrNumRatelimiters)); CHKiRet(statsobj.ConstructFinalize(modStats)); ENDmodInit diff --git a/runtime/hashtable/README b/runtime/hashtable/README new file mode 100644 index 00000000..5cadde0c --- /dev/null +++ b/runtime/hashtable/README @@ -0,0 +1,11 @@ +This is the hashtable code provided by +Christopher Clark +available at http://www.cl.cam.ac.uk/~cwc22/hashtable/ + +It may be slightly modified. The plan is to streamline +the code based on our needs and "really" integrate it into +the rsyslog runtime library. For the time being, we use it from +inside this subdirectory. We do not need all files, but I thought +I keep them together in case we later need something else. + +rgerhards, 2010-09-28 diff --git a/tools/Makefile.am b/tools/Makefile.am index 96657ad4..6541194a 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -36,7 +36,7 @@ rsyslogd_SOURCES = \ \ ../dirty.h rsyslogd_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) +rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) -lm rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS -- cgit v1.2.3 From 054d2ccdd6044f94823f8facbda935cb70646333 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 17:26:28 +0200 Subject: imuxsock: added per-socket hash tables/rate limiters & severity filter rate limiting now applies only to messages with a given severity or above. By default, emergency messages are NOT rate-limited. --- ChangeLog | 1 + plugins/imuxsock/imuxsock.c | 159 +++++++++++++++++++++++++++++--------------- runtime/rsyslog.h | 1 + tests/syslog_caller.c | 20 ++++++ 4 files changed, 126 insertions(+), 55 deletions(-) create mode 100644 tests/syslog_caller.c diff --git a/ChangeLog b/ChangeLog index 6622b67c..ca1428b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - added new config statements * $InputUnixListenSocketUsePIDFromSystem * $SystemLogUsePIDFromSystem +- imuxsock now supports up to 50 different sockets for input --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 728456b6..d500fc54 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -57,7 +57,7 @@ MODULE_TYPE_INPUT /* defines */ -#define MAXFUNIX 20 +#define MAXFUNIX 50 #ifndef _PATH_LOG #ifdef BSD #define _PATH_LOG "/var/run/log" @@ -94,8 +94,6 @@ struct rs_ratelimit_state { }; typedef struct rs_ratelimit_state rs_ratelimit_state_t; -/* support for per-process ratelimiting */ -static struct hashtable *hashtab; /* a very simple "hash function" for process IDs - we simply use the * pid itself: it is quite expected that all pids may log some time, but @@ -118,14 +116,19 @@ key_equals_fn(void *key1, void *key2) /* structure to describe a specific listener */ typedef struct lstn_s { - uchar *sockName; /* read-only after startup */ - prop_t *hostName; /* host-name override - if set, use this instead of actual name */ - int fd; /* read-only after startup */ - int flags; /* should parser parse host name? read-only after startup */ - int flowCtl; /* flow control settings for this socket */ - sbool bParseHost; /* should parser parse host name? read-only after startup */ - sbool bUseCreds; - sbool bCreatePath; /* auto-creation of socket directory? */ + uchar *sockName; /* read-only after startup */ + prop_t *hostName; /* host-name override - if set, use this instead of actual name */ + int fd; /* read-only after startup */ + int flags; /* should parser parse host name? read-only after startup */ + int flowCtl; /* flow control settings for this socket */ + int ratelimitInterval; + int ratelimitBurst; + intTiny ratelimitSev; /* severity level (and below) for which rate-limiting shall apply */ + struct hashtable *ht; /* our hashtable for rate-limiting */ + sbool bParseHost; /* should parser parse host name? read-only after startup */ + sbool bCreatePath; /* auto-creation of socket directory? */ + sbool bUseCreds; /* pull original creator credentials from socket */ + sbool bWritePid; /* write original PID into tag */ } lstn_t; static lstn_t listeners[MAXFUNIX]; @@ -142,15 +145,20 @@ static int bOmitLocalLogging = 0; static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ -static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ -static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ -static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bWritePid = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bWritePidSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ #define DFLT_ratelimitInterval 2 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ +static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 static int ratelimitBurst = DFLT_ratelimitBurst; /* max nbr of messages in interval */ +static int ratelimitBurstSysSock = DFLT_ratelimitBurst; /* max nbr of messages in interval */ +#define DFLT_ratelimitSeverity 1 /* do not rate-limit emergency messages */ +static int ratelimitSeverity = DFLT_ratelimitSeverity; +static int ratelimitSeveritySysSock = DFLT_ratelimitSeverity; @@ -192,7 +200,7 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) if(rs->missed) { snprintf((char*)msgbuf, sizeof(msgbuf), "imuxsock lost %u messages due to rate-limiting", rs->missed); - logmsgInternal(0, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); rs->missed = 0; } rs->begin = 0; @@ -262,11 +270,23 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) pLogHostName = NULL; } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); + if(ratelimitInterval > 0) { + if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + /* in this case, we simply turn of rate-limiting */ + dbgprintf("imuxsock: turning off rate limiting because we could not " + "create hash table\n"); + ratelimitInterval = 0; + } + } + listeners[nfd].ratelimitInterval = ratelimitInterval; + listeners[nfd].ratelimitBurst = ratelimitBurst; + listeners[nfd].ratelimitSev = ratelimitSeverity; listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; listeners[nfd].bCreatePath = bCreatePath; listeners[nfd].sockName = pNewVal; - listeners[nfd].bUseCreds = bUseCreds; + listeners[nfd].bUseCreds = (bWritePid || ratelimitInterval) ? 1 : 0; + listeners[nfd].bWritePid = bWritePid; nfd++; } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", @@ -278,11 +298,10 @@ finalize_it: } -/* free the sockName[] socket names - needed as cleanup on several places - * note that nfd is NOT reset! sockName[0] is never freed, as it comes from +/* discard all log sockets except for "socket" 0. Data for it comes from * the constant memory pool - and if not, it is freeed via some other pointer. */ -static rsRetVal discardFunixn(void) +static rsRetVal discardLogSockets(void) { int i; @@ -294,6 +313,9 @@ static rsRetVal discardFunixn(void) if(listeners[i].hostName != NULL) { prop.Destruct(&(listeners[i].hostName)); } + if(listeners[i].ht != NULL) { + hashtable_destroy(listeners[i].ht, 1); /* 1 => free all values automatically */ + } } return RS_RET_OK; @@ -410,7 +432,7 @@ finalize_it: * Returns NULL if not found. */ static inline rsRetVal -findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) +findRatelimiter(lstn_t *pLstn, struct ucred *cred, rs_ratelimit_state_t **prl) { rs_ratelimit_state_t *rl; int r; @@ -420,7 +442,7 @@ findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) if(cred == NULL) FINALIZE; - rl = hashtable_search(hashtab, &cred->pid); + rl = hashtable_search(pLstn->ht, &cred->pid); if(rl == NULL) { /* we need to add a new ratelimiter, process not seen before! */ dbgprintf("imuxsock: no ratelimiter for pid %lu, creating one\n", @@ -429,8 +451,8 @@ findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) CHKmalloc(rl = malloc(sizeof(rs_ratelimit_state_t))); CHKmalloc(keybuf = malloc(sizeof(pid_t))); *keybuf = cred->pid; - initRatelimitState(rl, ratelimitInterval, ratelimitBurst); - r = hashtable_insert(hashtab, keybuf, rl); + initRatelimitState(rl, ratelimitInterval, pLstn->ratelimitBurst); + r = hashtable_insert(pLstn->ht, keybuf, rl); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -482,13 +504,33 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) int i; uchar *parse; int pri; + int facil; + int sever; uchar bufParseTAG[CONF_TAG_MAXSIZE]; struct syslogTime st; time_t tt; - rs_ratelimit_state_t *ratelimiter; + rs_ratelimit_state_t *ratelimiter = NULL; DEFiRet; - findRatelimiter(cred, &ratelimiter); /* ignore error, better so than others... */ +// TODO: handle format errors?? + /* we need to parse the pri first, because we need the severity for + * rate-limiting as well. + */ + parse = pRcv; + lenMsg = lenRcv; + + parse++; lenMsg--; /* '<' */ + pri = 0; + while(lenMsg && isdigit(*parse)) { + pri = pri * 10 + *parse - '0'; + ++parse; + --lenMsg; + } + facil = LOG_FAC(pri); + sever = LOG_PRI(pri); + + if(sever >= pLstn->ratelimitSev) + findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ datetime.getCurrTime(&st, &tt); if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { @@ -502,26 +544,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); - -// TODO: handle format errors - parse = pRcv; - lenMsg = lenRcv; - - parse++; lenMsg--; /* '<' */ - pri = 0; - while(lenMsg && isdigit(*parse)) { - pri = pri * 10 + *parse - '0'; - ++parse; - --lenMsg; - } - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); + pMsg->iFacility = facil; + pMsg->iSeverity = sever; MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); parse++; lenMsg--; /* '>' */ if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { - dbgprintf("we have a problem, invalid timestamp in msg!\n"); + DBGPRINTF("we have a problem, invalid timestamp in msg!\n"); } /* pull tag */ @@ -532,7 +562,8 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) --lenMsg; } bufParseTAG[i] = '\0'; /* terminate string */ - fixPID(bufParseTAG, &i, cred); + if(pLstn->bWritePid) + fixPID(bufParseTAG, &i, cred); MsgSetTAG(pMsg, bufParseTAG, i); MsgSetMSGoffs(pMsg, lenRcv - lenMsg); @@ -716,7 +747,19 @@ CODESTARTwillRun # endif if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; - listeners[0].bUseCreds = bUseCredsSysSock; + if(ratelimitIntervalSysSock > 0) { + if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + /* in this case, we simply turn of rate-limiting */ + dbgprintf("imuxsock: turning off rate limiting because we could not " + "create hash table\n"); + ratelimitIntervalSysSock = 0; + } + } + listeners[0].ratelimitInterval = ratelimitIntervalSysSock; + listeners[0].ratelimitBurst = ratelimitBurstSysSock; + listeners[0].ratelimitSev = ratelimitSeveritySysSock; + listeners[0].bUseCreds = (bWritePidSysSock || ratelimitIntervalSysSock) ? 1 : 0; + listeners[0].bWritePid = bWritePidSysSock; /* initialize and return if will run or not */ actSocks = 0; @@ -724,7 +767,7 @@ CODESTARTwillRun if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); - } + } } if(actSocks == 0) { @@ -737,8 +780,6 @@ CODESTARTwillRun CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); - CHKmalloc(hashtab = create_hashtable(1000, hash_from_key_fn, key_equals_fn)); - finalize_it: ENDwillRun @@ -765,16 +806,12 @@ CODESTARTafterRun free(pLogSockName); free(pLogHostName); - discardFunixn(); + discardLogSockets(); nfd = 1; if(pInputName != NULL) prop.Destruct(&pInputName); - if(hashtab != NULL) { - hashtable_destroy(hashtab, 1); /* 1 => free all values automatically */ - hashtab = NULL; - } ENDafterRun @@ -815,15 +852,19 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a pLogHostName = NULL; } - discardFunixn(); + discardLogSockets(); nfd = 1; bIgnoreTimestamp = 1; bUseFlowCtl = 0; - bUseCreds = 0; - bUseCredsSysSock = 0; + bWritePid = 0; + bWritePidSysSock = 0; bCreatePath = DFLT_bCreatePath; ratelimitInterval = DFLT_ratelimitInterval; + ratelimitIntervalSysSock = DFLT_ratelimitInterval; ratelimitBurst = DFLT_ratelimitBurst; + ratelimitBurstSysSock = DFLT_ratelimitBurst; + ratelimitSeverity = DFLT_ratelimitSeverity; + ratelimitSeveritySysSock = DFLT_ratelimitSeverity; return RS_RET_OK; } @@ -881,13 +922,15 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); + NULL, &bWritePid, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitinterval", 0, eCmdHdlrInt, NULL, &ratelimitInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitburst", 0, eCmdHdlrInt, NULL, &ratelimitBurst, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitseverity", 0, eCmdHdlrInt, + NULL, &ratelimitSeverity, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); /* the following one is a (dirty) trick: the system log socket is not added via @@ -901,7 +944,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); + NULL, &bWritePidSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitinterval", 0, eCmdHdlrInt, + NULL, &ratelimitIntervalSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitburst", 0, eCmdHdlrInt, + NULL, &ratelimitBurstSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitseverity", 0, eCmdHdlrInt, + NULL, &ratelimitSeveritySysSock, STD_LOADABLE_MODULE_ID)); /* support statistics gathering */ CHKiRet(statsobj.Construct(&modStats)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index c58c259b..43203378 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -457,6 +457,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_EPOLL_CTL_FAILED = -2174, /**< epoll_ctl() failed */ RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ + RS_RET_RATE_LIMITED = -2177, /**< some messages discarded due to exceeding a rate limit */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/tests/syslog_caller.c b/tests/syslog_caller.c new file mode 100644 index 00000000..91a1f08b --- /dev/null +++ b/tests/syslog_caller.c @@ -0,0 +1,20 @@ +#include +#include +#include + +int main(int argc, char *argv[]) +{ + int i; + int sev = 0; + if(argc != 2) { + fprintf(stderr, "usage: syslog_caller num-messages\n"); + exit(1); + } + + int msgs = atoi(argv[1]); + + for(i = 0 ; i < msgs ; ++i) { + syslog(sev % 8, "test message nbr %d, severity=%d", i, sev % 8); + sev++; + } +} -- cgit v1.2.3 From f4d6418102033c22b0ae71d496e993b00773b5c4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 17:52:15 +0200 Subject: doc/imuxsock: added new options to doc --- ChangeLog | 14 +++++++++++++- doc/imuxsock.html | 38 ++++++++++++++++++++++++++++++++++++-- plugins/imuxsock/imuxsock.c | 2 +- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca1428b4..f9b79392 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,11 +1,23 @@ --------------------------------------------------------------------------- Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log - socket itself (thanks to Lennart Poettering for the suggestion) + socket itself + (thanks to Lennart Poettering for the suggesting this feature) +- imuxsock now optionally uses per-process input rate limiting, guarding the + user against processes spamming the system log + (thanks to Lennart Poettering for suggesting this feature) - added new config statements * $InputUnixListenSocketUsePIDFromSystem * $SystemLogUsePIDFromSystem + * $SystemLogRateLimitInterval + * $SystemLogRateLimitBurst + * $SystemLogRateLimitSeverity + * $IMUxSockRateLimitInterval + * $IMUxSockRateLimitBurst + * $IMUxSockRateLimitSeverity - imuxsock now supports up to 50 different sockets for input +- some code cleanup in imuxsock (consider this a release a major + modification, especially if problems show up) --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 4af2c030..f3ee7be2 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -25,6 +25,19 @@ the past four years. Alternate behaviour may be desirable if gateway-like processes send messages via the local log slot - in this case, it can be enabled via the $InputUnixListenSocketIgnoreMsgTimestamp and $SystemLogSocketIgnoreMsgTimestamp config directives

    +

    There is input rate limiting available, (since 5.7.1) to guard you against +the problems of a wild running logging process. +If more than $IMUXSockRateLimitInterval * $IMUXSockRateLimitBurst log messages are emitted +from the same process, those messages with $IMUXSockRateLimitSeverity or lower will be +dropped. It is not possible to recover anything about these messages, but imuxsock will +tell you how many it has dropped one the interval has expired AND the next message +is logged. Rate-limiting depends on SCM_CREDENTIALS. If the platform does not support +this socket option, rate limiting is turned off. If multiple sockets are configured, +rate limiting works independently on each of them (that should be what you usually expect). +The same functionality is available for the system log socket, which +just uses the prefix $SystemLogRateLimit... but otherwise works exactly the same. +When working with severities, please keep in mind that higher severity numbers mean lower +severity and configure things accordingly.

    Unix log sockets can be flow-controlled. That is, if processing queues fill up, the unix socket reader is blocked for a short while. This may be useful to prevent overruning the queues (which may cause exessive disk-io where it actually would not be needed). However, @@ -40,6 +53,15 @@ the implications. Note that for many systems, turning on flow control does not h
    Ignore timestamps included in the message. Applies to the next socket being added.

  • $InputUnixListenSocketFlowControl [on/off] - specifies if flow control should be applied to the next socket.
    • +
  • $IMUXSockRateLimitInterval [number] - specifies the rate-limiting +interval in seconds. Default value is 5 seconds. +
    • +
  • $IMUXSockRateLimitBurst [number] - specifies the rate-limiting +burst in number of messages. Default is 200. +
    • +
  • $IMUXSockRateLimitSeverity [numerical severity] - specifies the severity of +messages that shall be rate-limited. +
  • $InputUnixListenSocketUsePIDFromSystem [on/off] - specifies if the pid being logged shall be obtained from the log socket itself. If so, the TAG part of the message is rewritten. It is recommended to turn this option on, but the default is "off" to keep compatible @@ -54,6 +76,15 @@ to the system log socket.
    • be obtained from the log socket itself. If so, the TAG part of the message is rewritten. It is recommended to turn this option on, but the default is "off" to keep compatible with earlier versions of rsyslog. This option was introduced in 5.7.0. +
  • $SystemLogRateLimitInterval [number] - specifies the rate-limiting +interval in seconds. Default value is 5 seconds. +
    • +
  • $SystemLogRateLimitBurst [number] - specifies the rate-limiting +burst in number of messages. Default is 200. +
    • +
  • $SystemLogRateLimitSeverity [numerical severity] - specifies the severity of +messages that shall be rate-limited. +
  • $InputUnixListenSocketCreatePath [on/off] - create directories in the socket path if they do not already exist. They are created with 0755 permissions with the owner being the process under which rsyslogd runs. The default is not to create directories. Keep in mind, though, that rsyslogd always @@ -74,8 +105,11 @@ will only affect the next one and then automatically be reset. This functionalit that the local hostname can be overridden in cases where that is desired.
    • Caveats/Known Bugs:
    -
    -This documentation is sparse and incomplete. +
      +
    • There is a compile-time limit of 50 concurrent sockets. If you need more, you need to +change the array size in imuxsock.c. +
    • This documentation is sparse and incomplete. +

    Sample:

    The following sample is the minimum setup required to accept syslog messages from applications running on the local system.
    diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index d500fc54..b4ee8db0 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -150,7 +150,7 @@ static int bWritePid = 0; /* use credentials from recvmsg() and fixup PID in TA static int bWritePidSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ -#define DFLT_ratelimitInterval 2 +#define DFLT_ratelimitInterval 5 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 -- cgit v1.2.3 From c7e36e6b7e3e9fb24dfd82e2cb7683bfee0c15ad Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 18:12:40 +0200 Subject: build system: fixed some minor issues --- configure.ac | 2 +- runtime/Makefile.am | 2 + runtime/hashtable.h | 199 ++++++++++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.h | 199 ------------------------------------------ 4 files changed, 202 insertions(+), 200 deletions(-) create mode 100644 runtime/hashtable.h delete mode 100644 runtime/hashtable/hashtable.h diff --git a/configure.ac b/configure.ac index c52749c8..1050981b 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/runtime/Makefile.am b/runtime/Makefile.am index d657b6ca..5a0c4437 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -96,6 +96,8 @@ librsyslog_la_SOURCES = \ ../parse.h \ \ hashtable/hashtable.c \ + hashtable.h \ + hashtable/hashtable_private.h \ \ ../outchannel.c \ ../outchannel.h \ diff --git a/runtime/hashtable.h b/runtime/hashtable.h new file mode 100644 index 00000000..b90781ab --- /dev/null +++ b/runtime/hashtable.h @@ -0,0 +1,199 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_H__ +#define __HASHTABLE_CWC22_H__ + +struct hashtable; + +/* Example of use: + * + * struct hashtable *h; + * struct some_key *k; + * struct some_value *v; + * + * static unsigned int hash_from_key_fn( void *k ); + * static int keys_equal_fn ( void *key1, void *key2 ); + * + * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); + * k = (struct some_key *) malloc(sizeof(struct some_key)); + * v = (struct some_value *) malloc(sizeof(struct some_value)); + * + * (initialise k and v to suitable values) + * + * if (! hashtable_insert(h,k,v) ) + * { exit(-1); } + * + * if (NULL == (found = hashtable_search(h,k) )) + * { printf("not found!"); } + * + * if (NULL == (found = hashtable_remove(h,k) )) + * { printf("Not found\n"); } + * + */ + +/* Macros may be used to define type-safe(r) hashtable access functions, with + * methods specialized to take known key and value types as parameters. + * + * Example: + * + * Insert this at the start of your file: + * + * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); + * + * This defines the functions 'insert_some', 'search_some' and 'remove_some'. + * These operate just like hashtable_insert etc., with the same parameters, + * but their function signatures have 'struct some_key *' rather than + * 'void *', and hence can generate compile time errors if your program is + * supplying incorrect data as a key (and similarly for value). + * + * Note that the hash and key equality functions passed to create_hashtable + * still take 'void *' parameters instead of 'some key *'. This shouldn't be + * a difficult issue as they're only defined and passed once, and the other + * functions will ensure that only valid keys are supplied to them. + * + * The cost for this checking is increased code size and runtime overhead + * - if performance is important, it may be worth switching back to the + * unsafe methods once your program has been debugged with the safe methods. + * This just requires switching to some simple alternative defines - eg: + * #define insert_some hashtable_insert + * + */ + +/***************************************************************************** + * create_hashtable + + * @name create_hashtable + * @param minsize minimum initial size of hashtable + * @param hashfunction function for hashing keys + * @param key_eq_fn function for determining key equality + * @return newly created hashtable or NULL on failure + */ + +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashfunction) (void*), + int (*key_eq_fn) (void*,void*)); + +/***************************************************************************** + * hashtable_insert + + * @name hashtable_insert + * @param h the hashtable to insert into + * @param k the key - hashtable claims ownership and will free on removal + * @param v the value - does not claim ownership + * @return non-zero for successful insertion + * + * This function will cause the table to expand if the insertion would take + * the ratio of entries to table size over the maximum load factor. + * + * This function does not check for repeated insertions with a duplicate key. + * The value returned when using a duplicate key is undefined -- when + * the hashtable changes size, the order of retrieval of duplicate key + * entries is reversed. + * If in doubt, remove before insert. + */ + +int +hashtable_insert(struct hashtable *h, void *k, void *v); + +#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ +int fnname (struct hashtable *h, keytype *k, valuetype *v) \ +{ \ + return hashtable_insert(h,k,v); \ +} + +/***************************************************************************** + * hashtable_search + + * @name hashtable_search + * @param h the hashtable to search + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * +hashtable_search(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_search(h,k)); \ +} + +/***************************************************************************** + * hashtable_remove + + * @name hashtable_remove + * @param h the hashtable to remove the item from + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * /* returns value */ +hashtable_remove(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_remove(h,k)); \ +} + + +/***************************************************************************** + * hashtable_count + + * @name hashtable_count + * @param h the hashtable + * @return the number of items stored in the hashtable + */ +unsigned int +hashtable_count(struct hashtable *h); + + +/***************************************************************************** + * hashtable_destroy + + * @name hashtable_destroy + * @param h the hashtable + * @param free_values whether to call 'free' on the remaining values + */ + +void +hashtable_destroy(struct hashtable *h, int free_values); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.h b/runtime/hashtable/hashtable.h deleted file mode 100644 index b90781ab..00000000 --- a/runtime/hashtable/hashtable.h +++ /dev/null @@ -1,199 +0,0 @@ -/* Copyright (C) 2002 Christopher Clark */ - -#ifndef __HASHTABLE_CWC22_H__ -#define __HASHTABLE_CWC22_H__ - -struct hashtable; - -/* Example of use: - * - * struct hashtable *h; - * struct some_key *k; - * struct some_value *v; - * - * static unsigned int hash_from_key_fn( void *k ); - * static int keys_equal_fn ( void *key1, void *key2 ); - * - * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); - * k = (struct some_key *) malloc(sizeof(struct some_key)); - * v = (struct some_value *) malloc(sizeof(struct some_value)); - * - * (initialise k and v to suitable values) - * - * if (! hashtable_insert(h,k,v) ) - * { exit(-1); } - * - * if (NULL == (found = hashtable_search(h,k) )) - * { printf("not found!"); } - * - * if (NULL == (found = hashtable_remove(h,k) )) - * { printf("Not found\n"); } - * - */ - -/* Macros may be used to define type-safe(r) hashtable access functions, with - * methods specialized to take known key and value types as parameters. - * - * Example: - * - * Insert this at the start of your file: - * - * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); - * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); - * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); - * - * This defines the functions 'insert_some', 'search_some' and 'remove_some'. - * These operate just like hashtable_insert etc., with the same parameters, - * but their function signatures have 'struct some_key *' rather than - * 'void *', and hence can generate compile time errors if your program is - * supplying incorrect data as a key (and similarly for value). - * - * Note that the hash and key equality functions passed to create_hashtable - * still take 'void *' parameters instead of 'some key *'. This shouldn't be - * a difficult issue as they're only defined and passed once, and the other - * functions will ensure that only valid keys are supplied to them. - * - * The cost for this checking is increased code size and runtime overhead - * - if performance is important, it may be worth switching back to the - * unsafe methods once your program has been debugged with the safe methods. - * This just requires switching to some simple alternative defines - eg: - * #define insert_some hashtable_insert - * - */ - -/***************************************************************************** - * create_hashtable - - * @name create_hashtable - * @param minsize minimum initial size of hashtable - * @param hashfunction function for hashing keys - * @param key_eq_fn function for determining key equality - * @return newly created hashtable or NULL on failure - */ - -struct hashtable * -create_hashtable(unsigned int minsize, - unsigned int (*hashfunction) (void*), - int (*key_eq_fn) (void*,void*)); - -/***************************************************************************** - * hashtable_insert - - * @name hashtable_insert - * @param h the hashtable to insert into - * @param k the key - hashtable claims ownership and will free on removal - * @param v the value - does not claim ownership - * @return non-zero for successful insertion - * - * This function will cause the table to expand if the insertion would take - * the ratio of entries to table size over the maximum load factor. - * - * This function does not check for repeated insertions with a duplicate key. - * The value returned when using a duplicate key is undefined -- when - * the hashtable changes size, the order of retrieval of duplicate key - * entries is reversed. - * If in doubt, remove before insert. - */ - -int -hashtable_insert(struct hashtable *h, void *k, void *v); - -#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ -int fnname (struct hashtable *h, keytype *k, valuetype *v) \ -{ \ - return hashtable_insert(h,k,v); \ -} - -/***************************************************************************** - * hashtable_search - - * @name hashtable_search - * @param h the hashtable to search - * @param k the key to search for - does not claim ownership - * @return the value associated with the key, or NULL if none found - */ - -void * -hashtable_search(struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ -valuetype * fnname (struct hashtable *h, keytype *k) \ -{ \ - return (valuetype *) (hashtable_search(h,k)); \ -} - -/***************************************************************************** - * hashtable_remove - - * @name hashtable_remove - * @param h the hashtable to remove the item from - * @param k the key to search for - does not claim ownership - * @return the value associated with the key, or NULL if none found - */ - -void * /* returns value */ -hashtable_remove(struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ -valuetype * fnname (struct hashtable *h, keytype *k) \ -{ \ - return (valuetype *) (hashtable_remove(h,k)); \ -} - - -/***************************************************************************** - * hashtable_count - - * @name hashtable_count - * @param h the hashtable - * @return the number of items stored in the hashtable - */ -unsigned int -hashtable_count(struct hashtable *h); - - -/***************************************************************************** - * hashtable_destroy - - * @name hashtable_destroy - * @param h the hashtable - * @param free_values whether to call 'free' on the remaining values - */ - -void -hashtable_destroy(struct hashtable *h, int free_values); - -#endif /* __HASHTABLE_CWC22_H__ */ - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -- cgit v1.2.3 From 829a446c9f00e2f7016f09e18358458e2cc02189 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 29 Sep 2010 12:44:04 +0200 Subject: improved imuxsock doc & added small testing tool permanently --- doc/imuxsock.html | 9 +++++---- tests/Makefile.am | 5 ++++- tests/syslog_caller.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/doc/imuxsock.html b/doc/imuxsock.html index f3ee7be2..1ab99a76 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -27,15 +27,16 @@ case, it can be enabled via the $InputUnixListenSocketIgnoreMsgTimestamp and $SystemLogSocketIgnoreMsgTimestamp config directives

    There is input rate limiting available, (since 5.7.1) to guard you against the problems of a wild running logging process. -If more than $IMUXSockRateLimitInterval * $IMUXSockRateLimitBurst log messages are emitted -from the same process, those messages with $IMUXSockRateLimitSeverity or lower will be +If more than $SystemLogRateLimitInterval * $SystemLogRateLimitBurst log messages are emitted +from the same process, those messages with $SystemLogRateLimitSeverity or lower will be dropped. It is not possible to recover anything about these messages, but imuxsock will tell you how many it has dropped one the interval has expired AND the next message is logged. Rate-limiting depends on SCM_CREDENTIALS. If the platform does not support this socket option, rate limiting is turned off. If multiple sockets are configured, rate limiting works independently on each of them (that should be what you usually expect). -The same functionality is available for the system log socket, which -just uses the prefix $SystemLogRateLimit... but otherwise works exactly the same. +The same functionality is available for additional log sockets, in which case the +config statements just use +the prefix $IMUXSockRateLimit... but otherwise works exactly the same. When working with severities, please keep in mind that higher severity numbers mean lower severity and configure things accordingly.

    Unix log sockets can be flow-controlled. That is, if processing queues fill up, diff --git a/tests/Makefile.am b/tests/Makefile.am index 8538140e..62d45228 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller TESTS = $(TESTRUNS) cfg.sh \ arrayqueue.sh \ linkedlistqueue.sh \ @@ -322,6 +322,9 @@ chkseq_SOURCES = chkseq.c tcpflood_SOURCES = tcpflood.c tcpflood_LDADD = $(SOL_LIBS) +syslog_caller_SOURCES = syslog_caller.c +syslog_caller_LDADD = $(SOL_LIBS) + diagtalker_SOURCES = diagtalker.c diagtalker_LDADD = $(SOL_LIBS) diff --git a/tests/syslog_caller.c b/tests/syslog_caller.c index 91a1f08b..60f24166 100644 --- a/tests/syslog_caller.c +++ b/tests/syslog_caller.c @@ -1,3 +1,31 @@ +/* A very primitive testing tool that just emits a number of + * messages to the system log socket. Currently sufficient, but + * obviously room for improvement. + * + * It is highly suggested NOT to "base" any derivative work + * on this tool ;) + * + * Part of the testbench for rsyslog. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ #include #include #include -- cgit v1.2.3 From 33888b84e7dc6ec4341e066c68c7077cade24a7b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 29 Sep 2010 14:31:44 +0200 Subject: imuxsock: indicate when rate limiting begins, include pid in rl messages also improved test tool --- plugins/imuxsock/imuxsock.c | 13 ++++++++++--- tests/syslog_caller.c | 41 ++++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index b4ee8db0..24bcebb7 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -180,7 +180,7 @@ initRatelimitState(struct rs_ratelimit_state *rs, unsigned short interval, unsig * be called concurrently. */ static inline int -withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) +withinRatelimit(struct rs_ratelimit_state *rs, time_t tt, pid_t pid) { int ret; uchar msgbuf[1024]; @@ -199,7 +199,8 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) if(tt > rs->begin + rs->interval) { if(rs->missed) { snprintf((char*)msgbuf, sizeof(msgbuf), - "imuxsock lost %u messages due to rate-limiting", rs->missed); + "imuxsock lost %u messages from pid %lu due to rate-limiting", + rs->missed, (unsigned long) pid); logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); rs->missed = 0; } @@ -212,6 +213,12 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) rs->done++; ret = 1; } else { + if(rs->missed == 0) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "imuxsock begins to drop messages from pid %lu due to rate-limiting", + (unsigned long) pid); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + } rs->missed++; ret = 0; } @@ -533,7 +540,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ datetime.getCurrTime(&st, &tt); - if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { + if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt, cred->pid)) { STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); FINALIZE; } diff --git a/tests/syslog_caller.c b/tests/syslog_caller.c index 60f24166..3f2702a6 100644 --- a/tests/syslog_caller.c +++ b/tests/syslog_caller.c @@ -5,6 +5,11 @@ * It is highly suggested NOT to "base" any derivative work * on this tool ;) * + * Options + * + * -s severity (0..7 accoding to syslog spec, r "rolling", default 6) + * -m number of messages to generate (default 500) + * * Part of the testbench for rsyslog. * * Copyright 2010 Rainer Gerhards and Adiscon GmbH. @@ -28,21 +33,43 @@ */ #include #include +#include #include +static void usage(void) +{ + fprintf(stderr, "usage: syslog_caller num-messages\n"); + exit(1); +} + + int main(int argc, char *argv[]) { int i; - int sev = 0; - if(argc != 2) { - fprintf(stderr, "usage: syslog_caller num-messages\n"); - exit(1); - } + int opt; + int bRollingSev = 0; + int sev = 6; + int msgs = 500; - int msgs = atoi(argv[1]); + while((opt = getopt(argc, argv, "m:s:")) != -1) { + switch (opt) { + case 's': if(*optarg == 'r') { + bRollingSev = 1; + sev = 0; + } else + sev = atoi(optarg); + break; + case 'm': msgs = atoi(optarg); + break; + default: usage(); + break; + } + } for(i = 0 ; i < msgs ; ++i) { syslog(sev % 8, "test message nbr %d, severity=%d", i, sev % 8); - sev++; + if(bRollingSev) + sev++; } + return(0); } -- cgit v1.2.3 From 11617d8ab5192c12e3b33cd9c08ac32f1d334a85 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 30 Sep 2010 14:17:34 +0200 Subject: omhdfs: first shot at this new module (very rough PoC code) --- Makefile.am | 4 + configure.ac | 19 +++ plugins/omhdfs/Makefile.am | 6 + plugins/omhdfs/omhdfs.c | 296 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 plugins/omhdfs/Makefile.am create mode 100644 plugins/omhdfs/omhdfs.c diff --git a/Makefile.am b/Makefile.am index bc6d8dd0..c7646e3d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -127,6 +127,10 @@ if ENABLE_OMUXSOCK SUBDIRS += plugins/omuxsock endif +if ENABLE_OMHDFS +SUBDIRS += plugins/omhdfs +endif + if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif diff --git a/configure.ac b/configure.ac index 1050981b..069a8fd8 100644 --- a/configure.ac +++ b/configure.ac @@ -1003,6 +1003,23 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to search for omtemplate to find everything! +# settings for the omhdfs; +AC_ARG_ENABLE(omhdfs, + [AS_HELP_STRING([--enable-omhdfs],[Compiles omhdfs template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omhdfs="yes" ;; + no) enable_omhdfs="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omhdfs) ;; + esac], + [enable_omhdfs=no] +) +# +# you may want to do some library checks here - see snmp, mysql, pgsql modules +# for samples +# +AM_CONDITIONAL(ENABLE_OMHDFS, test x$enable_omhdfs = xyes) + + AC_CONFIG_FILES([Makefile \ runtime/Makefile \ tools/Makefile \ @@ -1016,6 +1033,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ plugins/omtemplate/Makefile \ + plugins/omhdfs/Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/pmrfc3164sd/Makefile \ @@ -1070,6 +1088,7 @@ echo "---{ output plugins }---" echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" +echo " omhdfs module will be compiled: $enable_omhdfs" echo " omruleset module will be compiled: $enable_omruleset" echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am new file mode 100644 index 00000000..329e6816 --- /dev/null +++ b/plugins/omhdfs/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = omhdfs.la + +omhdfs_la_SOURCES = omhdfs.c +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +omhdfs_la_LDFLAGS = -module -avoid-version +omhdfs_la_LIBADD = diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c new file mode 100644 index 00000000..42e5dc8a --- /dev/null +++ b/plugins/omhdfs/omhdfs.c @@ -0,0 +1,296 @@ +/* omhdfs.c + * This is the implementation of the build-in file output module. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +/* this is kind of a hack, if defined, it instructs omhdfs to use + * the regular (non-hdfs) file system calls. This eases development + * (and hopefully troubleshooting) especially in cases when no + * hdfs environment is available. + */ +#define USE_REGULAR_FS 1 + +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "conf.h" +#include "cfsysline.h" +#include "module-template.h" +#include "unicode-helper.h" +#ifndef USE_REGULAR_FS +# include "hdfs.h" +#endif + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* globals for default values */ +static uchar *fileName = NULL; +/* end globals for default values */ + +typedef struct _instanceData { + uchar *fileName; +# ifdef USE_REGULAR_FS + short fd; /* file descriptor for (current) file */ +# else + hdfsFS fs; + hdfsFile fd; +# endif +} instanceData; + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + printf("omhdfs: file:%s", pData->fileName); + if (pData->fd == -1) + printf(" (unused)"); +ENDdbgPrintInstInfo + + + +#if 0 +static void prepareFile(instanceData *pData, uchar *newFileName) +{ + if(access((char*)newFileName, F_OK) == 0) { + /* file already exists */ + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fCreateMode); + } else { + pData->fd = -1; + /* file does not exist, create it (and eventually parent directories */ + if(pData->bCreateDirs) { + /* we fist need to create parent dirs if they are missing + * We do not report any errors here ourselfs but let the code + * fall through to error handler below. + */ + if(makeFileParentDirs(newFileName, strlen((char*)newFileName), + pData->fDirCreateMode, pData->dirUID, + pData->dirGID, pData->bFailOnChown) != 0) { + return; /* we give up */ + } + } + /* no matter if we needed to create directories or not, we now try to create + * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) + */ + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fCreateMode); + if(pData->fd != -1) { + /* check and set uid/gid */ + if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { + /* we need to set owner/group */ + if(fchown(pData->fd, pData->fileUID, + pData->fileGID) != 0) { + if(pData->bFailOnChown) { + int eSave = errno; + close(pData->fd); + pData->fd = -1; + errno = eSave; + } + /* we will silently ignore the chown() failure + * if configured to do so. + */ + } + } + } + } +} +#endif + +static void prepareFile(instanceData *pData, uchar *newFileName) +{ +# if USE_REGULAR_FS + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); +# else + pData->fs = hdfsConnect("default", 0); + pData->fd = hdfsOpenFile(fs, newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + if(!pData->fd) { + dbgprintf(stderr, "Failed to open %s for writing!\n", newFileName); + // TODO: suspend/error report + } + +# endif +} + +static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +{ + size_t lenWrite; + DEFiRet; + +# if USE_REGULAR_FS + if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { + int e = errno; + dbgprintf("omhdfs write error!\n"); + + /* If the filesystem is filled up, just ignore + * it for now and continue writing when possible + */ + //if(pData->fileType == eTypeFILE && e == ENOSPC) + //return RS_RET_OK; + + //(void) close(pData->fd); + iRet = RS_RET_DISABLE_ACTION; + errno = e; + //logerror((char*) pData->f_fname); + } +# else + lenWrite = strlen(char*ppString[0]); + tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); + if(num_written_bytes != lenWrite) { + dbgprintf("Failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, + lenWrite, num_written_bytes); + // TODO: suspend/error report + } +# endif +else { dbgprintf("omhdfs has successfully written to file\n"); } + + RETiRet; +} + + +BEGINcreateInstance +CODESTARTcreateInstance + pData->fd = -1; +ENDcreateInstance + + +BEGINfreeInstance +CODESTARTfreeInstance +# ifdef USE_REGULAR_FS + if(pData->fd != -1) + close(pData->fd); +# else + hdfsCloseFile(pData->fs, pData->fd); +# endif +ENDfreeInstance + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + dbgprintf(" (%s)\n", pData->fileName); + iRet = writeFile(ppString, iMsgOpts, pData); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct + + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omhdfs:", sizeof(":omhdfs:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* rgerhards 2004-11-17: from now, we need to have different + * processing, because after the first comma, the template name + * to use is specified. So we need to scan for the first coma first + * and then look at the rest of the line. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); + //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); + + // TODO: check for NULL filename + CHKmalloc(pData->fileName = ustrdup(fileName)); + + prepareFile(pData, pData->fileName); + +#if 0 + if ( pData->fd < 0 ){ + pData->fd = -1; + dbgprintf("Error opening log file: %s\n", pData->f_fname); + logerror((char*) pData->f_fname); + } +#endif +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +/* Reset config variables for this module to default values. + * rgerhards, 2007-07-17 + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ +/* + fileUID = -1; + fileGID = -1; + dirUID = -1; + dirGID = -1; + bFailOnChown = 1; + iDynaFileCacheSize = 10; + fCreateMode = 0644; + fDirCreateMode = 0700; + bCreateDirs = 1; +*/ + return RS_RET_OK; +} + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit -- cgit v1.2.3 From 7ac7ad166b82034eea4a37c1937ca5ddd618ec45 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 30 Sep 2010 13:55:26 +0000 Subject: omhdfs: added "real" libhdfs code, now actually works on hdfs very crude implementation, but probably good enough to gather some early performance data and experience with the module. No real error handling done, if something breaks, the whole thing will be blown up ;) --- plugins/omhdfs/Makefile.am | 4 +-- plugins/omhdfs/omhdfs.c | 62 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 329e6816..23fcf75a 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -omhdfs_la_LDFLAGS = -module -avoid-version +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux +omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64 -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64/server -ljava -ljvm -lverify omhdfs_la_LIBADD = diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 42e5dc8a..fcdff6e5 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -28,7 +28,7 @@ * (and hopefully troubleshooting) especially in cases when no * hdfs environment is available. */ -#define USE_REGULAR_FS 1 +//#define USE_REGULAR_FS 1 #include "config.h" #include "rsyslog.h" @@ -62,6 +62,8 @@ DEF_OMOD_STATIC_DATA /* globals for default values */ static uchar *fileName = NULL; +static uchar *hdfsHost = NULL; +int hdfsPort = 0; /* end globals for default values */ typedef struct _instanceData { @@ -71,6 +73,8 @@ typedef struct _instanceData { # else hdfsFS fs; hdfsFile fd; + const char *hdfsHost; + tPort hdfsPort; # endif } instanceData; @@ -85,8 +89,8 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo printf("omhdfs: file:%s", pData->fileName); - if (pData->fd == -1) - printf(" (unused)"); + //if (pData->fd == -1) + //printf(" (unused)"); ENDdbgPrintInstInfo @@ -139,22 +143,41 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } #endif -static void prepareFile(instanceData *pData, uchar *newFileName) +static void +prepareFile(instanceData *pData, uchar *newFileName) { # if USE_REGULAR_FS pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); # else - pData->fs = hdfsConnect("default", 0); - pData->fd = hdfsOpenFile(fs, newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + dbgprintf("omhdfs: try to connect to host '%s' at port %d\n", + pData->hdfsHost, pData->hdfsPort); + pData->fs = hdfsConnect(pData->hdfsHost, pData->hdfsPort); + if(pData->fs == NULL) { + dbgprintf("omhdfs: error can not connect to hdfs\n"); + } + pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_APPEND, 0, 0, 0); + if(pData->fd == NULL) { + /* maybe the file does not exist, so we try to create it now. + * Note that we can not use hdfsExists() because of a deficit in + * it: https://issues.apache.org/jira/browse/HDFS-1154 + * As of my testing, libhdfs at least seems to return ENOENT if + * the file does not exist. + */ + if(errno == ENOENT) { + dbgprintf("omhdfs: ENOENT trying to append to '%s', now trying create\n", + newFileName); + pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + } + } if(!pData->fd) { - dbgprintf(stderr, "Failed to open %s for writing!\n", newFileName); + dbgprintf("omhdfs: failed to open %s for writing!\n", newFileName); // TODO: suspend/error report } # endif } -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal writeFile(uchar **ppString, instanceData *pData) { size_t lenWrite; DEFiRet; @@ -176,15 +199,14 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa //logerror((char*) pData->f_fname); } # else - lenWrite = strlen(char*ppString[0]); + lenWrite = strlen((char*) ppString[0]); tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); - if(num_written_bytes != lenWrite) { - dbgprintf("Failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, - lenWrite, num_written_bytes); + if((unsigned) num_written_bytes != lenWrite) { + dbgprintf("omhdfs: failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, + lenWrite, (unsigned long) num_written_bytes); // TODO: suspend/error report } # endif -else { dbgprintf("omhdfs has successfully written to file\n"); } RETiRet; } @@ -192,7 +214,7 @@ else { dbgprintf("omhdfs has successfully written to file\n"); } BEGINcreateInstance CODESTARTcreateInstance - pData->fd = -1; + //pData->fd = -1; ENDcreateInstance @@ -214,7 +236,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction dbgprintf(" (%s)\n", pData->fileName); - iRet = writeFile(ppString, iMsgOpts, pData); + iRet = writeFile(ppString, pData); ENDdoAction @@ -241,6 +263,12 @@ CODESTARTparseSelectorAct // TODO: check for NULL filename CHKmalloc(pData->fileName = ustrdup(fileName)); + if(hdfsHost == NULL) { + pData->hdfsHost = "default"; + } else { + CHKmalloc(pData->hdfsHost = strdup((char*)hdfsHost)); + } + pData->hdfsPort = hdfsPort; prepareFile(pData, pData->fileName); @@ -271,6 +299,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a fDirCreateMode = 0700; bCreateDirs = 1; */ + hdfsHost = NULL; + hdfsPort = 0; return RS_RET_OK; } @@ -291,6 +321,8 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr ENDmodInit -- cgit v1.2.3 From 1d609fc5ab9b5d1c89d5d7d8f321c4e8493b4437 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 10:27:44 +0200 Subject: omhdfs: some cleanup of build system but still pretty ugly. Any java folks out there to help clean it up? --- doc/omhdfs.html | 49 +++++++++++++++++++++++++++++++++++++++++++ doc/rsyslog_conf_modules.html | 1 + plugins/omhdfs/Makefile.am | 5 +++-- 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 doc/omhdfs.html diff --git a/doc/omhdfs.html b/doc/omhdfs.html new file mode 100644 index 00000000..827697b6 --- /dev/null +++ b/doc/omhdfs.html @@ -0,0 +1,49 @@ + +rsyslog output module for HDFS (omhdfs) +back + + +

    Unix sockets Output Module (omhdfs)

    +

    Module Name:    omhdfs

    +

    Available since:    5.7.2

    +

    Author: Rainer Gerhards <rgerhards@adiscon.com>

    +

    Description:

    +

    This module supports writing message into files on Hadoop's HDFS +file system. +

    Configuration Directives:

    +
      +
    • $...
      +option... +
      • +
    +Caveats/Known Bugs: +

    Building omhdfs is a challenge because we could not yet find out how +to integrate Java properly into the autotools build process. The issue is +that HDFS is written in Java and libhdfs uses JNI to talk to it. That requires +that various system-specific environment options and pathes be set correctly. At +this point, we leave this to the user. If someone know how to do it better, +please drop us a line! +

    In order to build, you need to set these environment variables BEFORE running +./configure: +

      +
    • JAVA_INCLUDES - must have all include pathes that are needed to build +JNI C programms, including the -I options necessary for gcc. An example is
      +# export JAVA_INCLUDES="-I/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include -I/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include/linux" +
    • JAVA_LIBS - must have all library pathes that are needed to build +JNI C programms, including the -l/-L options necessary for gcc. An example is
      +# export export JAVA_LIBS="-L/usr/java/jdk1.6.0_21/jre/lib/amd64 -L/usr/java/jdk1.6.0_21/jre/lib/amd64/server -ljava -ljvm -lverify" + +
    +

    Sample:

    +

    +

    + +[manual index] [rsyslog site]

    +

    This documentation is part of the rsyslog +project.
    +Copyright © 2010 by Rainer Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

    + + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 85899954..74aa319c 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -66,6 +66,7 @@ permits rsyslog to alert folks by mail if something important happens
  • omoracle - output module for Oracle (native OCI interface)
  • omudpspoof - output module sending UDP syslog messages with a spoofed address
  • omuxsock - output module Unix domain sockets
    • +
  • omhdfs - output module for Hadoop's HDFS file system

    • Parser Modules

    diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 23fcf75a..2e7ef8ea 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,6 +1,7 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux -omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64 -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64/server -ljava -ljvm -lverify +#omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(JAVA_INCLUDES) +omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs $(JAVA_LIBS) omhdfs_la_LIBADD = -- cgit v1.2.3 From d8a1489f545179591abced679ba24831d85ca224 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 11:29:50 +0200 Subject: omhdfs: cleanup and lots of improvement now things look much better, also done some prep in order to support a file cache (we need this for multiple selectors writing to the same file). --- plugins/omhdfs/omhdfs.c | 247 +++++++++++++++++++++++++++++------------------- runtime/rsyslog.h | 3 + 2 files changed, 153 insertions(+), 97 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index fcdff6e5..71080585 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -23,13 +23,6 @@ * A copy of the GPL can be found in the file "COPYING" in this distribution. */ -/* this is kind of a hack, if defined, it instructs omhdfs to use - * the regular (non-hdfs) file system calls. This eases development - * (and hopefully troubleshooting) especially in cases when no - * hdfs environment is available. - */ -//#define USE_REGULAR_FS 1 - #include "config.h" #include "rsyslog.h" #include @@ -42,6 +35,8 @@ #include #include #include +#include +#include #include "syslogd-types.h" #include "srUtils.h" @@ -50,15 +45,14 @@ #include "cfsysline.h" #include "module-template.h" #include "unicode-helper.h" -#ifndef USE_REGULAR_FS -# include "hdfs.h" -#endif +#include "errmsg.h" MODULE_TYPE_OUTPUT /* internal structures */ DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) /* globals for default values */ static uchar *fileName = NULL; @@ -66,18 +60,23 @@ static uchar *hdfsHost = NULL; int hdfsPort = 0; /* end globals for default values */ -typedef struct _instanceData { - uchar *fileName; -# ifdef USE_REGULAR_FS - short fd; /* file descriptor for (current) file */ -# else +typedef struct { + uchar *name; hdfsFS fs; - hdfsFile fd; + hdfsFile fh; const char *hdfsHost; tPort hdfsPort; -# endif + int nUsers; + pthread_mutex_t mut; +} file_t; + + +typedef struct _instanceData { + file_t *pFile; } instanceData; +/* forward definitions (down here, need data types) */ +static inline rsRetVal fileClose(file_t *pFile); BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -88,9 +87,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - printf("omhdfs: file:%s", pData->fileName); - //if (pData->fd == -1) - //printf(" (unused)"); + printf("omhdfs: file:%s", pData->pFile->name); ENDdbgPrintInstInfo @@ -100,10 +97,10 @@ static void prepareFile(instanceData *pData, uchar *newFileName) { if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, pData->fCreateMode); } else { - pData->fd = -1; + pData->fh = -1; /* file does not exist, create it (and eventually parent directories */ if(pData->bCreateDirs) { /* we fist need to create parent dirs if they are missing @@ -119,18 +116,18 @@ static void prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, pData->fCreateMode); - if(pData->fd != -1) { + if(pData->fh != -1) { /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, + if(fchown(pData->fh, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; - close(pData->fd); - pData->fd = -1; + close(pData->fh); + pData->fh = -1; errno = eSave; } /* we will silently ignore the chown() failure @@ -143,20 +140,71 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } #endif -static void -prepareFile(instanceData *pData, uchar *newFileName) +/* ---BEGIN FILE OBJECT---------------------------------------------------- */ +/* This code handles the "file object". This is split from the actual + * instance data, because several instances may write into the same file. + * If so, we need to use a single object, and also synchronize their writes. + * So we keep the file object separately, and just stick a reference into + * the instance data. + */ + +static inline rsRetVal +fileObjConstruct(file_t **ppFile) +{ + file_t *pFile; + DEFiRet; + + CHKmalloc(pFile = malloc(sizeof(file_t))); + pFile->name = NULL; + pFile->hdfsHost = NULL; + pFile->fh = NULL; + pFile->nUsers = 0; + + *ppFile = pFile; +finalize_it: + RETiRet; +} + +static inline void +fileObjAddUser(file_t *pFile) +{ + /* init mutex only when second user is added */ + ++pFile->nUsers; + if(pFile->nUsers == 2) + pthread_mutex_init(&pFile->mut, NULL); +} + +static inline rsRetVal +fileObjDestruct(file_t **ppFile) +{ + file_t *pFile = *ppFile; + if(pFile->nUsers > 1) + pthread_mutex_destroy(&pFile->mut); + fileClose(pFile); + free(pFile->name); + free((char*)pFile->hdfsHost); + free(pFile->fh); + + return RS_RET_OK; +} + +static inline rsRetVal +fileOpen(file_t *pFile) { -# if USE_REGULAR_FS - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); -# else - dbgprintf("omhdfs: try to connect to host '%s' at port %d\n", - pData->hdfsHost, pData->hdfsPort); - pData->fs = hdfsConnect(pData->hdfsHost, pData->hdfsPort); - if(pData->fs == NULL) { - dbgprintf("omhdfs: error can not connect to hdfs\n"); + DEFiRet; + + assert(pFile->fh == NULL); + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + DBGPRINTF("omhdfs: try to connect to HDFS at host '%s', port %d\n", + pFile->hdfsHost, pFile->hdfsPort); + pFile->fs = hdfsConnect(pFile->hdfsHost, pFile->hdfsPort); + if(pFile->fs == NULL) { + DBGPRINTF("omhdfs: error can not connect to hdfs\n"); + ABORT_FINALIZE(RS_RET_SUSPENDED); } - pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_APPEND, 0, 0, 0); - if(pData->fd == NULL) { + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_APPEND, 0, 0, 0); + if(pFile->fh == NULL) { /* maybe the file does not exist, so we try to create it now. * Note that we can not use hdfsExists() because of a deficit in * it: https://issues.apache.org/jira/browse/HDFS-1154 @@ -164,68 +212,75 @@ prepareFile(instanceData *pData, uchar *newFileName) * the file does not exist. */ if(errno == ENOENT) { - dbgprintf("omhdfs: ENOENT trying to append to '%s', now trying create\n", - newFileName); - pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + DBGPRINTF("omhdfs: ENOENT trying to append to '%s', now trying create\n", + pFile->name); + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_CREAT, 0, 0, 0); } } - if(!pData->fd) { - dbgprintf("omhdfs: failed to open %s for writing!\n", newFileName); - // TODO: suspend/error report + if(pFile->fh == NULL) { + DBGPRINTF("omhdfs: failed to open %s for writing!\n", pFile->name); + ABORT_FINALIZE(RS_RET_SUSPENDED); } -# endif +finalize_it: + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); + RETiRet; } -static rsRetVal writeFile(uchar **ppString, instanceData *pData) + +static inline rsRetVal +fileWrite(file_t *pFile, uchar *buf) { size_t lenWrite; DEFiRet; -# if USE_REGULAR_FS - if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { - int e = errno; - dbgprintf("omhdfs write error!\n"); - - /* If the filesystem is filled up, just ignore - * it for now and continue writing when possible - */ - //if(pData->fileType == eTypeFILE && e == ENOSPC) - //return RS_RET_OK; - - //(void) close(pData->fd); - iRet = RS_RET_DISABLE_ACTION; - errno = e; - //logerror((char*) pData->f_fname); - } -# else - lenWrite = strlen((char*) ppString[0]); - tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); + assert(pFile->fh != NULL); + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + lenWrite = strlen((char*) buf); + tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { - dbgprintf("omhdfs: failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, - lenWrite, (unsigned long) num_written_bytes); - // TODO: suspend/error report + errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, "omhdfs: failed to write %s, expected %lu bytes, " + "written %lu\n", pFile->name, lenWrite, (unsigned long) num_written_bytes); + ABORT_FINALIZE(RS_RET_SUSPENDED); } -# endif +finalize_it: + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); RETiRet; } +static inline rsRetVal +fileClose(file_t *pFile) +{ + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + if(pFile->fh != NULL) { + hdfsCloseFile(pFile->fs, pFile->fh); + pFile->fh = NULL; + } + + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); + + return RS_RET_OK; +} + +/* ---END FILE OBJECT---------------------------------------------------- */ + + BEGINcreateInstance CODESTARTcreateInstance - //pData->fd = -1; + pData->pFile = NULL; ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance -# ifdef USE_REGULAR_FS - if(pData->fd != -1) - close(pData->fd); -# else - hdfsCloseFile(pData->fs, pData->fd); -# endif + fileObjDestruct(&pData->pFile); ENDfreeInstance @@ -235,8 +290,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - dbgprintf(" (%s)\n", pData->fileName); - iRet = writeFile(ppString, pData); + DBGPRINTF(" (%s)\n", pData->pFile->name); + iRet = fileWrite(pData->pFile, ppString[0]); ENDdoAction @@ -251,34 +306,29 @@ CODESTARTparseSelectorAct /* ok, if we reach this point, we have something for us */ p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ CHKiRet(createInstance(&pData)); - CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* rgerhards 2004-11-17: from now, we need to have different - * processing, because after the first comma, the template name - * to use is specified. So we need to scan for the first coma first - * and then look at the rest of the line. - */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); - // TODO: check for NULL filename - CHKmalloc(pData->fileName = ustrdup(fileName)); + if(fileName == NULL) { + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: no file name specified, can not continue"); + ABORT_FINALIZE(RS_RET_FILE_NOT_SPECIFIED); + } + + CHKiRet(fileObjConstruct(&pData->pFile)); if(hdfsHost == NULL) { - pData->hdfsHost = "default"; + CHKmalloc(pData->pFile->hdfsHost = strdup("default")); } else { - CHKmalloc(pData->hdfsHost = strdup((char*)hdfsHost)); + CHKmalloc(pData->pFile->hdfsHost = strdup((char*)hdfsHost)); } - pData->hdfsPort = hdfsPort; + pData->pFile->hdfsPort = hdfsPort; - prepareFile(pData, pData->fileName); + fileOpen(pData->pFile); -#if 0 - if ( pData->fd < 0 ){ - pData->fd = -1; - dbgprintf("Error opening log file: %s\n", pData->f_fname); - logerror((char*) pData->f_fname); + if(pData->pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pData->pFile->name); + iRet = RS_RET_SUSPENDED; } -#endif CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -307,6 +357,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINmodExit CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -320,6 +371,8 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 43203378..7ccc9cb8 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -458,6 +458,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ RS_RET_RATE_LIMITED = -2177, /**< some messages discarded due to exceeding a rate limit */ + RS_RET_ERR_HDFS_WRITE = -2178, /**< error writing to HDFS */ + RS_RET_ERR_HDFS_OPEN = -2179, /**< error during hdfsOpen (e.g. file does not exist) */ + RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 0ee524c391d017225049542fffe572d7de7d1512 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 09:55:21 +0000 Subject: omhdfs: fixed small bug ... that I could only see on my execution environment (I right now have two envs, a local one where I can compile, but not run and a remote one where I can do both, but this is a bit less convenient to use). --- plugins/omhdfs/omhdfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 71080585..5d28d5dd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -316,6 +316,7 @@ CODESTARTparseSelectorAct } CHKiRet(fileObjConstruct(&pData->pFile)); + CHKmalloc(pData->pFile->name = (uchar*)strdup((char*)fileName)); if(hdfsHost == NULL) { CHKmalloc(pData->pFile->hdfsHost = strdup("default")); } else { -- cgit v1.2.3 From 9696cdef34f5d033564138fb9d4afb87daa6b1be Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 12:32:01 +0200 Subject: omhdfs: files now kept inside a hashtable for use by multiple actions Note:compiles, but not yet tested --- plugins/omhdfs/omhdfs.c | 40 +++++++++++++++++++++++++++++----------- runtime/hashtable.h | 4 +++- runtime/hashtable/hashtable.c | 27 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 5d28d5dd..eaca90e4 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -46,6 +46,7 @@ #include "module-template.h" #include "unicode-helper.h" #include "errmsg.h" +#include "hashtable.h" MODULE_TYPE_OUTPUT @@ -54,6 +55,9 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +/* global data */ +static struct hashtable *files; /* holds all file objects that we know */ + /* globals for default values */ static uchar *fileName = NULL; static uchar *hdfsHost = NULL; @@ -172,6 +176,7 @@ fileObjAddUser(file_t *pFile) ++pFile->nUsers; if(pFile->nUsers == 2) pthread_mutex_init(&pFile->mut, NULL); + dbgprintf("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); } static inline rsRetVal @@ -296,6 +301,9 @@ ENDdoAction BEGINparseSelectorAct + file_t *pFile; + int r; + uchar *keybuf; CODESTARTparseSelectorAct /* first check if this config line is actually for us */ @@ -315,21 +323,28 @@ CODESTARTparseSelectorAct ABORT_FINALIZE(RS_RET_FILE_NOT_SPECIFIED); } - CHKiRet(fileObjConstruct(&pData->pFile)); - CHKmalloc(pData->pFile->name = (uchar*)strdup((char*)fileName)); - if(hdfsHost == NULL) { - CHKmalloc(pData->pFile->hdfsHost = strdup("default")); - } else { - CHKmalloc(pData->pFile->hdfsHost = strdup((char*)hdfsHost)); + pFile = hashtable_search(files, fileName); + if(pFile == NULL) { + /* we need a new file object, this one not seen before */ + CHKiRet(fileObjConstruct(&pFile)); + CHKmalloc(pFile->name = (uchar*)strdup((char*)fileName)); + CHKmalloc(keybuf = ustrdup(fileName)); + r = hashtable_insert(files, keybuf, pFile); + if(r == 0) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } - pData->pFile->hdfsPort = hdfsPort; + fileObjAddUser(pFile); - fileOpen(pData->pFile); - - if(pData->pFile->fh == NULL){ - errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pData->pFile->name); + CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); + pFile->hdfsPort = hdfsPort; + fileOpen(pFile); + if(pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); iRet = RS_RET_SUSPENDED; } + + pData->pFile = pFile; + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -359,6 +374,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINmodExit CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); + if(files != NULL) + hashtable_destroy(files, 1); /* 1 => free all values automatically */ ENDmodExit @@ -373,6 +390,7 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); diff --git a/runtime/hashtable.h b/runtime/hashtable.h index b90781ab..0f980127 100644 --- a/runtime/hashtable.h +++ b/runtime/hashtable.h @@ -196,4 +196,6 @@ hashtable_destroy(struct hashtable *h, int free_values); * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ + */ +unsigned int hash_from_string(void *k) ; +int key_equals_string(void *key1, void *key2); diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c index a10e3bc6..e2a2b3f4 100644 --- a/runtime/hashtable/hashtable.c +++ b/runtime/hashtable/hashtable.c @@ -241,6 +241,33 @@ hashtable_destroy(struct hashtable *h, int free_values) free(h); } +/* some generic hash functions */ + +/* one provided by Aaaron Wiebe based on perl's hashng algorithm + * (so probably pretty generic). Not for excessively large strings! + */ +unsigned int +hash_from_string(void *k) +{ + int len; + char *rkey = (char*) k; + unsigned hashval = 1; + + len = (int) strlen(rkey); + while (len--) + hashval = hashval * 33 + *rkey++; + + return hashval; +} + + +int +key_equals_string(void *key1, void *key2) +{ + return strcmp(key1, key2); +} + + /* * Copyright (c) 2002, Christopher Clark * All rights reserved. -- cgit v1.2.3 From 255895a58b3f2a54fecf971da700caf265b4e1f0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 13:59:48 +0000 Subject: omhdfs: more improvements finally this looks almost production ready for files where no directory path needs to be created --- plugins/imuxsock/imuxsock.c | 4 +-- plugins/omhdfs/Makefile.am | 3 +-- plugins/omhdfs/omhdfs.c | 47 ++++++++++++++++++++++++----------- runtime/hashtable.h | 3 ++- runtime/hashtable/hashtable.c | 17 ++++++++++--- runtime/hashtable/hashtable_private.h | 1 + 6 files changed, 53 insertions(+), 22 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 24bcebb7..ad2d61c8 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -278,7 +278,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); if(ratelimitInterval > 0) { - if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); @@ -755,7 +755,7 @@ CODESTARTwillRun if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; if(ratelimitIntervalSysSock > 0) { - if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 2e7ef8ea..95e6b102 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,7 +1,6 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -#omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(JAVA_INCLUDES) omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs $(JAVA_LIBS) -omhdfs_la_LIBADD = +omhdfs_la_LIBADD = $(RSRT_LIBS) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index eaca90e4..734c28cd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -61,6 +61,7 @@ static struct hashtable *files; /* holds all file objects that we know */ /* globals for default values */ static uchar *fileName = NULL; static uchar *hdfsHost = NULL; +static uchar *dfltTplName = NULL; /* default template name to use */ int hdfsPort = 0; /* end globals for default values */ @@ -176,10 +177,10 @@ fileObjAddUser(file_t *pFile) ++pFile->nUsers; if(pFile->nUsers == 2) pthread_mutex_init(&pFile->mut, NULL); - dbgprintf("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); + DBGPRINTF("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); } -static inline rsRetVal +static rsRetVal fileObjDestruct(file_t **ppFile) { file_t *pFile = *ppFile; @@ -193,6 +194,18 @@ fileObjDestruct(file_t **ppFile) return RS_RET_OK; } +/* this function is to be used as destructor for the + * hash table code. + */ +static void +fileObjDestruct4Hashtable(void *ptr) +{ + dbgprintf("omfile: fileObjDestruct4Hashtable called\n"); + file_t *pFile = (file_t*) ptr; + fileObjDestruct(&pFile); +} + + static inline rsRetVal fileOpen(file_t *pFile) { @@ -261,6 +274,8 @@ finalize_it: static inline rsRetVal fileClose(file_t *pFile) { + DEFiRet; + if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); if(pFile->fh != NULL) { @@ -271,7 +286,7 @@ fileClose(file_t *pFile) if(pFile->nUsers > 1) d_pthread_mutex_unlock(&pFile->mut); - return RS_RET_OK; + RETiRet; } /* ---END FILE OBJECT---------------------------------------------------- */ @@ -285,7 +300,8 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - fileObjDestruct(&pData->pFile); + if(pData->pFile != NULL) + fileObjDestruct(&pData->pFile); ENDfreeInstance @@ -315,8 +331,8 @@ CODESTARTparseSelectorAct p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ CHKiRet(createInstance(&pData)); CODE_STD_STRING_REQUESTparseSelectorAct(1) - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); - //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, + (dfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : dfltTplName)); if(fileName == NULL) { errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: no file name specified, can not continue"); @@ -327,17 +343,18 @@ CODESTARTparseSelectorAct if(pFile == NULL) { /* we need a new file object, this one not seen before */ CHKiRet(fileObjConstruct(&pFile)); - CHKmalloc(pFile->name = (uchar*)strdup((char*)fileName)); + CHKmalloc(pFile->name = fileName); CHKmalloc(keybuf = ustrdup(fileName)); + fileName = NULL; /* re-set, data passed to file object */ + CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); + pFile->hdfsPort = hdfsPort; + fileOpen(pFile); r = hashtable_insert(files, keybuf, pFile); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } fileObjAddUser(pFile); - CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); - pFile->hdfsPort = hdfsPort; - fileOpen(pFile); if(pFile->fh == NULL){ errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); iRet = RS_RET_SUSPENDED; @@ -390,11 +407,13 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); - CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string)); + CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string, + fileObjDestruct4Hashtable)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsdefaulttemplate", 0, eCmdHdlrGetWord, NULL, &dfltTplName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr ENDmodInit diff --git a/runtime/hashtable.h b/runtime/hashtable.h index 0f980127..f777ad0b 100644 --- a/runtime/hashtable.h +++ b/runtime/hashtable.h @@ -68,13 +68,14 @@ struct hashtable; * @param minsize minimum initial size of hashtable * @param hashfunction function for hashing keys * @param key_eq_fn function for determining key equality + * @param dest destructor for value entries (NULL -> use free()) * @return newly created hashtable or NULL on failure */ struct hashtable * create_hashtable(unsigned int minsize, unsigned int (*hashfunction) (void*), - int (*key_eq_fn) (void*,void*)); + int (*key_eq_fn) (void*,void*), void (*dest) (void*)); /***************************************************************************** * hashtable_insert diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c index e2a2b3f4..41fc60fe 100644 --- a/runtime/hashtable/hashtable.c +++ b/runtime/hashtable/hashtable.c @@ -29,7 +29,7 @@ const float max_load_factor = 0.65; struct hashtable * create_hashtable(unsigned int minsize, unsigned int (*hashf) (void*), - int (*eqf) (void*,void*)) + int (*eqf) (void*,void*), void (*dest)(void*)) { struct hashtable *h; unsigned int pindex, size = primes[0]; @@ -49,6 +49,7 @@ create_hashtable(unsigned int minsize, h->entrycount = 0; h->hashfn = hashf; h->eqfn = eqf; + h->dest = dest; h->loadlimit = (unsigned int) ceil(size * max_load_factor); return h; } @@ -225,7 +226,16 @@ hashtable_destroy(struct hashtable *h, int free_values) { e = table[i]; while (NULL != e) - { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } + { + f = e; + e = e->next; + freekey(f->k); + if(h->dest == NULL) + free(f->v); + else + h->dest(f->v); + free(f); + } } } else @@ -264,7 +274,8 @@ hash_from_string(void *k) int key_equals_string(void *key1, void *key2) { - return strcmp(key1, key2); + /* we must return true IF the keys are equal! */ + return !strcmp(key1, key2); } diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h index 3e95f600..10b82da4 100644 --- a/runtime/hashtable/hashtable_private.h +++ b/runtime/hashtable/hashtable_private.h @@ -21,6 +21,7 @@ struct hashtable { unsigned int primeindex; unsigned int (*hashfn) (void *k); int (*eqfn) (void *k1, void *k2); + void (*dest) (void *v); /* destructor for values, if NULL use free() */ }; /*****************************************************************************/ -- cgit v1.2.3 From 670e81c9a8275a5509efd71dae66d9a267ec1574 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 14:36:39 +0000 Subject: omhdfs: made action suspend/resume working --- plugins/omhdfs/omhdfs.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 734c28cd..4fbf2ef4 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -276,16 +276,19 @@ fileClose(file_t *pFile) { DEFiRet; + if(pFile->fh == NULL) + FINALIZE; + if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); - if(pFile->fh != NULL) { - hdfsCloseFile(pFile->fs, pFile->fh); - pFile->fh = NULL; - } + + hdfsCloseFile(pFile->fs, pFile->fh); + pFile->fh = NULL; if(pFile->nUsers > 1) d_pthread_mutex_unlock(&pFile->mut); +finalize_it: RETiRet; } @@ -307,12 +310,26 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + fileClose(pData->pFile); + fileOpen(pData->pFile); + if(pData->pFile->fh == NULL){ + dbgprintf("omhdfs: tried to resume file %s, but still no luck...\n", + pData->pFile->name); + iRet = RS_RET_SUSPENDED; + } ENDtryResume BEGINdoAction CODESTARTdoAction DBGPRINTF(" (%s)\n", pData->pFile->name); + if(pData->pFile->fh == NULL) { + fileOpen(pData->pFile); + if(pData->pFile->fh == NULL) { + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } iRet = fileWrite(pData->pFile, ppString[0]); +finalize_it: ENDdoAction @@ -349,17 +366,16 @@ CODESTARTparseSelectorAct CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); pFile->hdfsPort = hdfsPort; fileOpen(pFile); + if(pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - " + "retrying later", pFile->name); + iRet = RS_RET_SUSPENDED; + } r = hashtable_insert(files, keybuf, pFile); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } fileObjAddUser(pFile); - - if(pFile->fh == NULL){ - errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); - iRet = RS_RET_SUSPENDED; - } - pData->pFile = pFile; CODE_STD_FINALIZERparseSelectorAct -- cgit v1.2.3 From d5f16404f93d54afddebb9fb683469fc712d2335 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 17:26:14 +0200 Subject: omhdfs: added doc --- ChangeLog | 3 +++ doc/omhdfs.html | 28 ++++++++++++++++++++++++---- plugins/omhdfs/omhdfs.c | 2 +- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index f9b79392..8129f634 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-09-?? +- support for Hadoop's HDFS added (via omhdfs) +--------------------------------------------------------------------------- Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log socket itself diff --git a/doc/omhdfs.html b/doc/omhdfs.html index 827697b6..3849f167 100644 --- a/doc/omhdfs.html +++ b/doc/omhdfs.html @@ -12,8 +12,19 @@ file system.

    Configuration Directives:

      -
    • $...
      -option... +
    • $OMHDFSFileName [name]
      +The name of the file to which the output data shall be written. +
      • +
    • $OMHDFSHost [name]
      +Name or IP address of the HDFS host to connect to. +
      • +
    • $OMHDFSPort [name]
      +Port on which to connect to the HDFS host. +
      • +
    • $OMHDFSDefaultTemplate [name]
      +Default template to be used when none is specified. This saves the work of +specifying the same template ever and ever again. Of course, the default +template can be overwritten via the usual method.
    Caveats/Known Bugs: @@ -23,7 +34,8 @@ that HDFS is written in Java and libhdfs uses JNI to talk to it. That requires that various system-specific environment options and pathes be set correctly. At this point, we leave this to the user. If someone know how to do it better, please drop us a line! -

    In order to build, you need to set these environment variables BEFORE running +

      +
    • In order to build, you need to set these environment variables BEFORE running ./configure:
      • JAVA_INCLUDES - must have all include pathes that are needed to build @@ -32,12 +44,20 @@ JNI C programms, including the -I options necessary for gcc. An example is
      • JAVA_LIBS - must have all library pathes that are needed to build JNI C programms, including the -l/-L options necessary for gcc. An example is
        # export export JAVA_LIBS="-L/usr/java/jdk1.6.0_21/jre/lib/amd64 -L/usr/java/jdk1.6.0_21/jre/lib/amd64/server -ljava -ljvm -lverify" +
      +
    • As of HDFS architecture, you must make sure that all relevant environment +variables (the usual Java stuff and HADOOP's home directory) are properly set. +
    • As it looks, libhdfs makes Java throw exceptions to stdout. There is no +known work-around for this (and it usually should not case any troubles.

    Sample:

    - [manual index] [rsyslog site]

    This documentation is part of the rsyslog diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 4fbf2ef4..42ed834f 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -1,5 +1,5 @@ /* omhdfs.c - * This is the implementation of the build-in file output module. + * This is an output module to support Hadoop's HDFS. * * NOTE: read comments in module-template.h to understand how this file * works! -- cgit v1.2.3 From 2cc5fbd49fdca20cdeb8eb81455bc1baef627a77 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 09:43:34 +0200 Subject: bugfix/imuxsock: system log socket was deleted in systemd-mode, too related bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=200 Note that this may not be the ultimate fix. I could not definitely identify the root cause of the problem and so I did some changes that I *think* (but could not verify) that resolved the issue. --- plugins/imuxsock/imuxsock.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 24bcebb7..906e28e6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -139,6 +139,7 @@ static int startIndexUxLocalSockets; /* process fd from that index on (used to * read-only after startup */ static int nfd = 1; /* number of Unix sockets open / read-only after startup */ +static int bSysSockFromSystemd = 0; /* Did we receive the system socket from systemd? */ /* config settings */ static int bOmitLocalLogging = 0; @@ -368,6 +369,7 @@ openLogSocket(lstn_t *pLstn) return -1; if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { + bSysSockFromSystemd = 1; /* set default */ int r; /* System log socket code. Check whether an FD was passed in from systemd. If @@ -396,6 +398,7 @@ openLogSocket(lstn_t *pLstn) errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } + bSysSockFromSystemd = 1; /* indicate we got the socket from systemd */ } else { CHKiRet(createLogSocket(pLstn)); } @@ -415,15 +418,6 @@ openLogSocket(lstn_t *pLstn) } } -#if 0 - // SO_TIMESTAMP currently does not work for an unknown reason. Any help is appreciated! - one = 1; - if(setsockopt(pLstn->fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", pLstn->sockName); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } -#endif - finalize_it: if(iRet != RS_RET_OK) { close(pLstn->fd); @@ -802,13 +796,16 @@ CODESTARTafterRun /* Clean-up files. If systemd passed us a socket it is * systemd's job to clean it up.*/ - if (sd_listen_fds(0) > 0) + if(bSysSockFromSystemd) { + DBGPRINTF("imuxsock: got system socket from systemd, not unlinking it\n"); i = 1; - else + } else i = startIndexUxLocalSockets; for(; i < nfd; i++) - if (listeners[i].sockName && listeners[i].fd != -1) + if (listeners[i].sockName && listeners[i].fd != -1) { + DBGPRINTF("imuxsock: unlinking unix socket file[%d] %s\n", i, listeners[i].sockName); unlink((char*) listeners[i].sockName); + } /* free no longer needed string */ free(pLogSockName); free(pLogHostName); -- cgit v1.2.3 From 8e51241a50531dc10d00e54aee00a24fc6760811 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Sep 2010 02:01:10 +0200 Subject: systemd: install service and socket unit files This adds a systemd socket and service unit file to the default install if systemd is found or explicitly enabled in ./configure. Patch is against current git v5-devel. --- .gitignore | 2 ++ Makefile.am | 25 +++++++++++++++++++++++-- configure.ac | 10 ++++++++++ rsyslog.service.in | 8 ++++++++ rsyslog.socket | 8 ++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 rsyslog.service.in create mode 100644 rsyslog.socket diff --git a/.gitignore b/.gitignore index bc362e7f..55f069ee 100644 --- a/.gitignore +++ b/.gitignore @@ -25,9 +25,11 @@ install-sh missing compile rsyslogd +rsyslog.service *.orig rg.conf* *.swp +*.cache # some common names I use during development utils tmp* diff --git a/Makefile.am b/Makefile.am index bc6d8dd0..f3dca447 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,6 +39,25 @@ lmgssutil_la_LDFLAGS = -module -avoid-version lmgssutil_la_LIBADD = $(GSS_LIBS) endif +# +# systemd support +# +if HAVE_SYSTEMD + +dist_systemdsystemunit_DATA = \ + rsyslog.socket + +nodist_systemdsystemunit_DATA = \ + rsyslog.service + +CLEANFILES = \ + rsyslog.service + +%.service: %.service.in + $(AM_V_GEN)sed -e 's,@sbindir\@,$(sbindir),g' $< > $@ + +endif + EXTRA_DIST = \ freebsd/rsyslogd \ slackware/rc.rsyslogd \ @@ -47,7 +66,8 @@ EXTRA_DIST = \ COPYING.LESSER \ contrib/gnutls/ca.pem \ contrib/gnutls/cert.pem \ - contrib/gnutls/key.pem + contrib/gnutls/key.pem \ + rsyslog.service.in SUBDIRS = doc runtime . plugins/immark plugins/imuxsock plugins/imtcp plugins/imudp plugins/omtesting @@ -196,5 +216,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-shave \ --enable-extended-tests \ --enable-impstats \ - --enable-memcheck + --enable-memcheck \ + --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 1050981b..16454d00 100644 --- a/configure.ac +++ b/configure.ac @@ -360,6 +360,16 @@ if test "$enable_unlimited_select" = "yes"; then fi +# support for systemd unit files +AC_ARG_WITH([systemdsystemunitdir], + AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files]), + [], [with_systemdsystemunitdir=$($PKG_CONFIG --variable=systemdsystemunitdir systemd)]) +if test "x$with_systemdsystemunitdir" != xno; then + AC_SUBST([systemdsystemunitdir], [$with_systemdsystemunitdir]) +fi +AM_CONDITIONAL(HAVE_SYSTEMD, [test -n "$with_systemdsystemunitdir" -a "x$with_systemdsystemunitdir" != xno ]) + + # debug AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], diff --git a/rsyslog.service.in b/rsyslog.service.in new file mode 100644 index 00000000..b660f6bc --- /dev/null +++ b/rsyslog.service.in @@ -0,0 +1,8 @@ +[Unit] +Description=System Logging Service + +[Service] +ExecStart=@sbindir@/rsyslogd -n -c 4 + +[Install] +WantedBy=multi-user.target diff --git a/rsyslog.socket b/rsyslog.socket new file mode 100644 index 00000000..0cd86054 --- /dev/null +++ b/rsyslog.socket @@ -0,0 +1,8 @@ +[Unit] +Description=Syslog Socket + +[Socket] +ListenDatagram=/dev/log + +[Install] +WantedBy=sockets.target -- cgit v1.2.3 From b3aa4c696a3e8dd97b549afcf49959173b30fe6a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 12:30:43 +0200 Subject: minor: corrected -c param in systemd startup files --- rsyslog.service.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsyslog.service.in b/rsyslog.service.in index b660f6bc..ea966cc3 100644 --- a/rsyslog.service.in +++ b/rsyslog.service.in @@ -2,7 +2,7 @@ Description=System Logging Service [Service] -ExecStart=@sbindir@/rsyslogd -n -c 4 +ExecStart=@sbindir@/rsyslogd -n -c5 [Install] WantedBy=multi-user.target -- cgit v1.2.3 From e40cb595a2da943483124fae8f215a397add9fca Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 13:58:41 +0200 Subject: omhdfs: support for HUP added --- plugins/omhdfs/omhdfs.c | 42 ++++- runtime/Makefile.am | 6 +- runtime/hashtable.c | 313 ++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.c | 313 ---------------------------------- runtime/hashtable/hashtable_itr.c | 188 -------------------- runtime/hashtable/hashtable_itr.h | 112 ------------ runtime/hashtable/hashtable_private.h | 86 ---------- runtime/hashtable_itr.c | 190 +++++++++++++++++++++ runtime/hashtable_itr.h | 112 ++++++++++++ runtime/hashtable_private.h | 86 ++++++++++ 10 files changed, 738 insertions(+), 710 deletions(-) create mode 100644 runtime/hashtable.c delete mode 100644 runtime/hashtable/hashtable.c delete mode 100644 runtime/hashtable/hashtable_itr.c delete mode 100644 runtime/hashtable/hashtable_itr.h delete mode 100644 runtime/hashtable/hashtable_private.h create mode 100644 runtime/hashtable_itr.c create mode 100644 runtime/hashtable_itr.h create mode 100644 runtime/hashtable_private.h diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 42ed834f..eefea722 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -47,6 +47,7 @@ #include "unicode-helper.h" #include "errmsg.h" #include "hashtable.h" +#include "hashtable_itr.h" MODULE_TYPE_OUTPUT @@ -200,7 +201,6 @@ fileObjDestruct(file_t **ppFile) static void fileObjDestruct4Hashtable(void *ptr) { - dbgprintf("omfile: fileObjDestruct4Hashtable called\n"); file_t *pFile = (file_t*) ptr; fileObjDestruct(&pFile); } @@ -214,6 +214,7 @@ fileOpen(file_t *pFile) assert(pFile->fh == NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); + DBGPRINTF("omhdfs: try to connect to HDFS at host '%s', port %d\n", pFile->hdfsHost, pFile->hdfsPort); pFile->fs = hdfsConnect(pFile->hdfsHost, pFile->hdfsPort); @@ -256,6 +257,17 @@ fileWrite(file_t *pFile, uchar *buf) assert(pFile->fh != NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); + + /* open file if not open. This must be done *here* and while mutex-protected + * because of HUP handling (which is async to normal processing!). + */ + if(pFile->fh == NULL) { + fileOpen(pFile); + if(pFile->fh == NULL) { + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } + lenWrite = strlen((char*) buf); tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { @@ -321,15 +333,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->pFile->name); - if(pData->pFile->fh == NULL) { - fileOpen(pData->pFile); - if(pData->pFile->fh == NULL) { - ABORT_FINALIZE(RS_RET_SUSPENDED); - } - } + DBGPRINTF("omuxsock: action to to write to %s\n", pData->pFile->name); iRet = fileWrite(pData->pFile, ppString[0]); -finalize_it: ENDdoAction @@ -382,6 +387,24 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct +BEGINdoHUP + file_t *pFile; + struct hashtable_itr *itr; +CODESTARTdoHUP + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(files); + if(hashtable_count(files) > 0) + { + do { + pFile = (file_t *) hashtable_iterator_value(itr); + fileClose(pFile); + DBGPRINTF("imuxsock: HUP, closing file %s\n", pFile->name); + } while (hashtable_iterator_advance(itr)); + } +ENDdoHUP + + /* Reset config variables for this module to default values. * rgerhards, 2007-07-17 */ @@ -415,6 +438,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_doHUP ENDqueryEtryPt diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 5a0c4437..93817e75 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -95,9 +95,11 @@ librsyslog_la_SOURCES = \ ../parse.c \ ../parse.h \ \ - hashtable/hashtable.c \ + hashtable.c \ hashtable.h \ - hashtable/hashtable_private.h \ + hashtable_itr.c \ + hashtable_itr.h \ + hashtable_private.h \ \ ../outchannel.c \ ../outchannel.h \ diff --git a/runtime/hashtable.c b/runtime/hashtable.c new file mode 100644 index 00000000..41fc60fe --- /dev/null +++ b/runtime/hashtable.c @@ -0,0 +1,313 @@ +/* Copyright (C) 2004 Christopher Clark */ +/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include +#include +#include +#include + +/* +Credit for primes table: Aaron Krowne + http://br.endernet.org/~akrowne/ + http://planetmath.org/encyclopedia/GoodHashTablePrimes.html +*/ +static const unsigned int primes[] = { +53, 97, 193, 389, +769, 1543, 3079, 6151, +12289, 24593, 49157, 98317, +196613, 393241, 786433, 1572869, +3145739, 6291469, 12582917, 25165843, +50331653, 100663319, 201326611, 402653189, +805306457, 1610612741 +}; +const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); +const float max_load_factor = 0.65; + +/*****************************************************************************/ +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashf) (void*), + int (*eqf) (void*,void*), void (*dest)(void*)) +{ + struct hashtable *h; + unsigned int pindex, size = primes[0]; + /* Check requested hashtable isn't too large */ + if (minsize > (1u << 30)) return NULL; + /* Enforce size as prime */ + for (pindex=0; pindex < prime_table_length; pindex++) { + if (primes[pindex] > minsize) { size = primes[pindex]; break; } + } + h = (struct hashtable *)malloc(sizeof(struct hashtable)); + if (NULL == h) return NULL; /*oom*/ + h->table = (struct entry **)malloc(sizeof(struct entry*) * size); + if (NULL == h->table) { free(h); return NULL; } /*oom*/ + memset(h->table, 0, size * sizeof(struct entry *)); + h->tablelength = size; + h->primeindex = pindex; + h->entrycount = 0; + h->hashfn = hashf; + h->eqfn = eqf; + h->dest = dest; + h->loadlimit = (unsigned int) ceil(size * max_load_factor); + return h; +} + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k) +{ + /* Aim to protect against poor hash functions by adding logic here + * - logic taken from java 1.4 hashtable source */ + unsigned int i = h->hashfn(k); + i += ~(i << 9); + i ^= ((i >> 14) | (i << 18)); /* >>> */ + i += (i << 4); + i ^= ((i >> 10) | (i << 22)); /* >>> */ + return i; +} + +/*****************************************************************************/ +static int +hashtable_expand(struct hashtable *h) +{ + /* Double the size of the table to accomodate more entries */ + struct entry **newtable; + struct entry *e; + struct entry **pE; + unsigned int newsize, i, idx; + /* Check we're not hitting max capacity */ + if (h->primeindex == (prime_table_length - 1)) return 0; + newsize = primes[++(h->primeindex)]; + + newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); + if (NULL != newtable) + { + memset(newtable, 0, newsize * sizeof(struct entry *)); + /* This algorithm is not 'stable'. ie. it reverses the list + * when it transfers entries between the tables */ + for (i = 0; i < h->tablelength; i++) { + while (NULL != (e = h->table[i])) { + h->table[i] = e->next; + idx = indexFor(newsize,e->h); + e->next = newtable[idx]; + newtable[idx] = e; + } + } + free(h->table); + h->table = newtable; + } + /* Plan B: realloc instead */ + else + { + newtable = (struct entry **) + realloc(h->table, newsize * sizeof(struct entry *)); + if (NULL == newtable) { (h->primeindex)--; return 0; } + h->table = newtable; + memset(newtable[h->tablelength], 0, newsize - h->tablelength); + for (i = 0; i < h->tablelength; i++) { + for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { + idx = indexFor(newsize,e->h); + if (idx == i) + { + pE = &(e->next); + } + else + { + *pE = e->next; + e->next = newtable[idx]; + newtable[idx] = e; + } + } + } + } + h->tablelength = newsize; + h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + return -1; +} + +/*****************************************************************************/ +unsigned int +hashtable_count(struct hashtable *h) +{ + return h->entrycount; +} + +/*****************************************************************************/ +int +hashtable_insert(struct hashtable *h, void *k, void *v) +{ + /* This method allows duplicate keys - but they shouldn't be used */ + unsigned int idx; + struct entry *e; + if (++(h->entrycount) > h->loadlimit) + { + /* Ignore the return value. If expand fails, we should + * still try cramming just this value into the existing table + * -- we may not have memory for a larger table, but one more + * element may be ok. Next time we insert, we'll try expanding again.*/ + hashtable_expand(h); + } + e = (struct entry *)malloc(sizeof(struct entry)); + if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ + e->h = hash(h,k); + idx = indexFor(h->tablelength,e->h); + e->k = k; + e->v = v; + e->next = h->table[idx]; + h->table[idx] = e; + return -1; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_search(struct hashtable *h, void *k) +{ + struct entry *e; + unsigned int hashvalue, idx; + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hashvalue); + e = h->table[idx]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_remove(struct hashtable *h, void *k) +{ + /* TODO: consider compacting the table when the load factor drops enough, + * or provide a 'compact' method. */ + + struct entry *e; + struct entry **pE; + void *v; + unsigned int hashvalue, idx; + + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hash(h,k)); + pE = &(h->table[idx]); + e = *pE; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + *pE = e->next; + h->entrycount--; + v = e->v; + freekey(e->k); + free(e); + return v; + } + pE = &(e->next); + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +/* destroy */ +void +hashtable_destroy(struct hashtable *h, int free_values) +{ + unsigned int i; + struct entry *e, *f; + struct entry **table = h->table; + if (free_values) + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { + f = e; + e = e->next; + freekey(f->k); + if(h->dest == NULL) + free(f->v); + else + h->dest(f->v); + free(f); + } + } + } + else + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f); } + } + } + free(h->table); + free(h); +} + +/* some generic hash functions */ + +/* one provided by Aaaron Wiebe based on perl's hashng algorithm + * (so probably pretty generic). Not for excessively large strings! + */ +unsigned int +hash_from_string(void *k) +{ + int len; + char *rkey = (char*) k; + unsigned hashval = 1; + + len = (int) strlen(rkey); + while (len--) + hashval = hashval * 33 + *rkey++; + + return hashval; +} + + +int +key_equals_string(void *key1, void *key2) +{ + /* we must return true IF the keys are equal! */ + return !strcmp(key1, key2); +} + + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c deleted file mode 100644 index 41fc60fe..00000000 --- a/runtime/hashtable/hashtable.c +++ /dev/null @@ -1,313 +0,0 @@ -/* Copyright (C) 2004 Christopher Clark */ -/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ - -#include "hashtable.h" -#include "hashtable_private.h" -#include -#include -#include -#include - -/* -Credit for primes table: Aaron Krowne - http://br.endernet.org/~akrowne/ - http://planetmath.org/encyclopedia/GoodHashTablePrimes.html -*/ -static const unsigned int primes[] = { -53, 97, 193, 389, -769, 1543, 3079, 6151, -12289, 24593, 49157, 98317, -196613, 393241, 786433, 1572869, -3145739, 6291469, 12582917, 25165843, -50331653, 100663319, 201326611, 402653189, -805306457, 1610612741 -}; -const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); -const float max_load_factor = 0.65; - -/*****************************************************************************/ -struct hashtable * -create_hashtable(unsigned int minsize, - unsigned int (*hashf) (void*), - int (*eqf) (void*,void*), void (*dest)(void*)) -{ - struct hashtable *h; - unsigned int pindex, size = primes[0]; - /* Check requested hashtable isn't too large */ - if (minsize > (1u << 30)) return NULL; - /* Enforce size as prime */ - for (pindex=0; pindex < prime_table_length; pindex++) { - if (primes[pindex] > minsize) { size = primes[pindex]; break; } - } - h = (struct hashtable *)malloc(sizeof(struct hashtable)); - if (NULL == h) return NULL; /*oom*/ - h->table = (struct entry **)malloc(sizeof(struct entry*) * size); - if (NULL == h->table) { free(h); return NULL; } /*oom*/ - memset(h->table, 0, size * sizeof(struct entry *)); - h->tablelength = size; - h->primeindex = pindex; - h->entrycount = 0; - h->hashfn = hashf; - h->eqfn = eqf; - h->dest = dest; - h->loadlimit = (unsigned int) ceil(size * max_load_factor); - return h; -} - -/*****************************************************************************/ -unsigned int -hash(struct hashtable *h, void *k) -{ - /* Aim to protect against poor hash functions by adding logic here - * - logic taken from java 1.4 hashtable source */ - unsigned int i = h->hashfn(k); - i += ~(i << 9); - i ^= ((i >> 14) | (i << 18)); /* >>> */ - i += (i << 4); - i ^= ((i >> 10) | (i << 22)); /* >>> */ - return i; -} - -/*****************************************************************************/ -static int -hashtable_expand(struct hashtable *h) -{ - /* Double the size of the table to accomodate more entries */ - struct entry **newtable; - struct entry *e; - struct entry **pE; - unsigned int newsize, i, idx; - /* Check we're not hitting max capacity */ - if (h->primeindex == (prime_table_length - 1)) return 0; - newsize = primes[++(h->primeindex)]; - - newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); - if (NULL != newtable) - { - memset(newtable, 0, newsize * sizeof(struct entry *)); - /* This algorithm is not 'stable'. ie. it reverses the list - * when it transfers entries between the tables */ - for (i = 0; i < h->tablelength; i++) { - while (NULL != (e = h->table[i])) { - h->table[i] = e->next; - idx = indexFor(newsize,e->h); - e->next = newtable[idx]; - newtable[idx] = e; - } - } - free(h->table); - h->table = newtable; - } - /* Plan B: realloc instead */ - else - { - newtable = (struct entry **) - realloc(h->table, newsize * sizeof(struct entry *)); - if (NULL == newtable) { (h->primeindex)--; return 0; } - h->table = newtable; - memset(newtable[h->tablelength], 0, newsize - h->tablelength); - for (i = 0; i < h->tablelength; i++) { - for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { - idx = indexFor(newsize,e->h); - if (idx == i) - { - pE = &(e->next); - } - else - { - *pE = e->next; - e->next = newtable[idx]; - newtable[idx] = e; - } - } - } - } - h->tablelength = newsize; - h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); - return -1; -} - -/*****************************************************************************/ -unsigned int -hashtable_count(struct hashtable *h) -{ - return h->entrycount; -} - -/*****************************************************************************/ -int -hashtable_insert(struct hashtable *h, void *k, void *v) -{ - /* This method allows duplicate keys - but they shouldn't be used */ - unsigned int idx; - struct entry *e; - if (++(h->entrycount) > h->loadlimit) - { - /* Ignore the return value. If expand fails, we should - * still try cramming just this value into the existing table - * -- we may not have memory for a larger table, but one more - * element may be ok. Next time we insert, we'll try expanding again.*/ - hashtable_expand(h); - } - e = (struct entry *)malloc(sizeof(struct entry)); - if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ - e->h = hash(h,k); - idx = indexFor(h->tablelength,e->h); - e->k = k; - e->v = v; - e->next = h->table[idx]; - h->table[idx] = e; - return -1; -} - -/*****************************************************************************/ -void * /* returns value associated with key */ -hashtable_search(struct hashtable *h, void *k) -{ - struct entry *e; - unsigned int hashvalue, idx; - hashvalue = hash(h,k); - idx = indexFor(h->tablelength,hashvalue); - e = h->table[idx]; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; - e = e->next; - } - return NULL; -} - -/*****************************************************************************/ -void * /* returns value associated with key */ -hashtable_remove(struct hashtable *h, void *k) -{ - /* TODO: consider compacting the table when the load factor drops enough, - * or provide a 'compact' method. */ - - struct entry *e; - struct entry **pE; - void *v; - unsigned int hashvalue, idx; - - hashvalue = hash(h,k); - idx = indexFor(h->tablelength,hash(h,k)); - pE = &(h->table[idx]); - e = *pE; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) - { - *pE = e->next; - h->entrycount--; - v = e->v; - freekey(e->k); - free(e); - return v; - } - pE = &(e->next); - e = e->next; - } - return NULL; -} - -/*****************************************************************************/ -/* destroy */ -void -hashtable_destroy(struct hashtable *h, int free_values) -{ - unsigned int i; - struct entry *e, *f; - struct entry **table = h->table; - if (free_values) - { - for (i = 0; i < h->tablelength; i++) - { - e = table[i]; - while (NULL != e) - { - f = e; - e = e->next; - freekey(f->k); - if(h->dest == NULL) - free(f->v); - else - h->dest(f->v); - free(f); - } - } - } - else - { - for (i = 0; i < h->tablelength; i++) - { - e = table[i]; - while (NULL != e) - { f = e; e = e->next; freekey(f->k); free(f); } - } - } - free(h->table); - free(h); -} - -/* some generic hash functions */ - -/* one provided by Aaaron Wiebe based on perl's hashng algorithm - * (so probably pretty generic). Not for excessively large strings! - */ -unsigned int -hash_from_string(void *k) -{ - int len; - char *rkey = (char*) k; - unsigned hashval = 1; - - len = (int) strlen(rkey); - while (len--) - hashval = hashval * 33 + *rkey++; - - return hashval; -} - - -int -key_equals_string(void *key1, void *key2) -{ - /* we must return true IF the keys are equal! */ - return !strcmp(key1, key2); -} - - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_itr.c b/runtime/hashtable/hashtable_itr.c deleted file mode 100644 index 5dced841..00000000 --- a/runtime/hashtable/hashtable_itr.c +++ /dev/null @@ -1,188 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#include "hashtable.h" -#include "hashtable_private.h" -#include "hashtable_itr.h" -#include /* defines NULL */ - -/*****************************************************************************/ -/* hashtable_iterator - iterator constructor */ - -struct hashtable_itr * -hashtable_iterator(struct hashtable *h) -{ - unsigned int i, tablelength; - struct hashtable_itr *itr = (struct hashtable_itr *) - malloc(sizeof(struct hashtable_itr)); - if (NULL == itr) return NULL; - itr->h = h; - itr->e = NULL; - itr->parent = NULL; - tablelength = h->tablelength; - itr->index = tablelength; - if (0 == h->entrycount) return itr; - - for (i = 0; i < tablelength; i++) - { - if (NULL != h->table[i]) - { - itr->e = h->table[i]; - itr->index = i; - break; - } - } - return itr; -} - -/*****************************************************************************/ -/* key - return the key of the (key,value) pair at the current position */ -/* value - return the value of the (key,value) pair at the current position */ - -void * -hashtable_iterator_key(struct hashtable_itr *i) -{ return i->e->k; } - -void * -hashtable_iterator_value(struct hashtable_itr *i) -{ return i->e->v; } - -/*****************************************************************************/ -/* advance - advance the iterator to the next element - * returns zero if advanced to end of table */ - -int -hashtable_iterator_advance(struct hashtable_itr *itr) -{ - unsigned int j,tablelength; - struct entry **table; - struct entry *next; - if (NULL == itr->e) return 0; /* stupidity check */ - - next = itr->e->next; - if (NULL != next) - { - itr->parent = itr->e; - itr->e = next; - return -1; - } - tablelength = itr->h->tablelength; - itr->parent = NULL; - if (tablelength <= (j = ++(itr->index))) - { - itr->e = NULL; - return 0; - } - table = itr->h->table; - while (NULL == (next = table[j])) - { - if (++j >= tablelength) - { - itr->index = tablelength; - itr->e = NULL; - return 0; - } - } - itr->index = j; - itr->e = next; - return -1; -} - -/*****************************************************************************/ -/* remove - remove the entry at the current iterator position - * and advance the iterator, if there is a successive - * element. - * If you want the value, read it before you remove: - * beware memory leaks if you don't. - * Returns zero if end of iteration. */ - -int -hashtable_iterator_remove(struct hashtable_itr *itr) -{ - struct entry *remember_e, *remember_parent; - int ret; - - /* Do the removal */ - if (NULL == (itr->parent)) - { - /* element is head of a chain */ - itr->h->table[itr->index] = itr->e->next; - } else { - /* element is mid-chain */ - itr->parent->next = itr->e->next; - } - /* itr->e is now outside the hashtable */ - remember_e = itr->e; - itr->h->entrycount--; - freekey(remember_e->k); - - /* Advance the iterator, correcting the parent */ - remember_parent = itr->parent; - ret = hashtable_iterator_advance(itr); - if (itr->parent == remember_e) { itr->parent = remember_parent; } - free(remember_e); - return ret; -} - -/*****************************************************************************/ -int /* returns zero if not found */ -hashtable_iterator_search(struct hashtable_itr *itr, - struct hashtable *h, void *k) -{ - struct entry *e, *parent; - unsigned int hashvalue, index; - - hashvalue = hash(h,k); - index = indexFor(h->tablelength,hashvalue); - - e = h->table[index]; - parent = NULL; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) - { - itr->index = index; - itr->e = e; - itr->parent = parent; - itr->h = h; - return -1; - } - parent = e; - e = e->next; - } - return 0; -} - - -/* - * Copyright (c) 2002, 2004, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_itr.h b/runtime/hashtable/hashtable_itr.h deleted file mode 100644 index eea699a7..00000000 --- a/runtime/hashtable/hashtable_itr.h +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#ifndef __HASHTABLE_ITR_CWC22__ -#define __HASHTABLE_ITR_CWC22__ -#include "hashtable.h" -#include "hashtable_private.h" /* needed to enable inlining */ - -/*****************************************************************************/ -/* This struct is only concrete here to allow the inlining of two of the - * accessor functions. */ -struct hashtable_itr -{ - struct hashtable *h; - struct entry *e; - struct entry *parent; - unsigned int index; -}; - - -/*****************************************************************************/ -/* hashtable_iterator - */ - -struct hashtable_itr * -hashtable_iterator(struct hashtable *h); - -/*****************************************************************************/ -/* hashtable_iterator_key - * - return the value of the (key,value) pair at the current position */ - -extern inline void * -hashtable_iterator_key(struct hashtable_itr *i) -{ - return i->e->k; -} - -/*****************************************************************************/ -/* value - return the value of the (key,value) pair at the current position */ - -extern inline void * -hashtable_iterator_value(struct hashtable_itr *i) -{ - return i->e->v; -} - -/*****************************************************************************/ -/* advance - advance the iterator to the next element - * returns zero if advanced to end of table */ - -int -hashtable_iterator_advance(struct hashtable_itr *itr); - -/*****************************************************************************/ -/* remove - remove current element and advance the iterator to the next element - * NB: if you need the value to free it, read it before - * removing. ie: beware memory leaks! - * returns zero if advanced to end of table */ - -int -hashtable_iterator_remove(struct hashtable_itr *itr); - -/*****************************************************************************/ -/* search - overwrite the supplied iterator, to point to the entry - * matching the supplied key. - h points to the hashtable to be searched. - * returns zero if not found. */ -int -hashtable_iterator_search(struct hashtable_itr *itr, - struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ -int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ -{ \ - return (hashtable_iterator_search(i,h,k)); \ -} - - - -#endif /* __HASHTABLE_ITR_CWC22__*/ - -/* - * Copyright (c) 2002, 2004, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h deleted file mode 100644 index 10b82da4..00000000 --- a/runtime/hashtable/hashtable_private.h +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#ifndef __HASHTABLE_PRIVATE_CWC22_H__ -#define __HASHTABLE_PRIVATE_CWC22_H__ - -#include "hashtable.h" - -/*****************************************************************************/ -struct entry -{ - void *k, *v; - unsigned int h; - struct entry *next; -}; - -struct hashtable { - unsigned int tablelength; - struct entry **table; - unsigned int entrycount; - unsigned int loadlimit; - unsigned int primeindex; - unsigned int (*hashfn) (void *k); - int (*eqfn) (void *k1, void *k2); - void (*dest) (void *v); /* destructor for values, if NULL use free() */ -}; - -/*****************************************************************************/ -unsigned int -hash(struct hashtable *h, void *k); - -/*****************************************************************************/ -/* indexFor */ -static inline unsigned int -indexFor(unsigned int tablelength, unsigned int hashvalue) { - return (hashvalue % tablelength); -}; - -/* Only works if tablelength == 2^N */ -/*static inline unsigned int -indexFor(unsigned int tablelength, unsigned int hashvalue) -{ - return (hashvalue & (tablelength - 1u)); -} -*/ - -/*****************************************************************************/ -#define freekey(X) free(X) -/*define freekey(X) ; */ - - -/*****************************************************************************/ - -#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable_itr.c b/runtime/hashtable_itr.c new file mode 100644 index 00000000..967287f1 --- /dev/null +++ b/runtime/hashtable_itr.c @@ -0,0 +1,190 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_itr.h" +#include /* defines NULL */ + +/*****************************************************************************/ +/* hashtable_iterator - iterator constructor */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h) +{ + unsigned int i, tablelength; + struct hashtable_itr *itr = (struct hashtable_itr *) + malloc(sizeof(struct hashtable_itr)); + if (NULL == itr) return NULL; + itr->h = h; + itr->e = NULL; + itr->parent = NULL; + tablelength = h->tablelength; + itr->index = tablelength; + if (0 == h->entrycount) return itr; + + for (i = 0; i < tablelength; i++) + { + if (NULL != h->table[i]) + { + itr->e = h->table[i]; + itr->index = i; + break; + } + } + return itr; +} + +/*****************************************************************************/ +/* key - return the key of the (key,value) pair at the current position */ +/* value - return the value of the (key,value) pair at the current position */ + +#if 0 /* these are now inline functions! */ +void * +hashtable_iterator_key(struct hashtable_itr *i) +{ return i->e->k; } + +void * +hashtable_iterator_value(struct hashtable_itr *i) +{ return i->e->v; } +#endif + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr) +{ + unsigned int j,tablelength; + struct entry **table; + struct entry *next; + if (NULL == itr->e) return 0; /* stupidity check */ + + next = itr->e->next; + if (NULL != next) + { + itr->parent = itr->e; + itr->e = next; + return -1; + } + tablelength = itr->h->tablelength; + itr->parent = NULL; + if (tablelength <= (j = ++(itr->index))) + { + itr->e = NULL; + return 0; + } + table = itr->h->table; + while (NULL == (next = table[j])) + { + if (++j >= tablelength) + { + itr->index = tablelength; + itr->e = NULL; + return 0; + } + } + itr->index = j; + itr->e = next; + return -1; +} + +/*****************************************************************************/ +/* remove - remove the entry at the current iterator position + * and advance the iterator, if there is a successive + * element. + * If you want the value, read it before you remove: + * beware memory leaks if you don't. + * Returns zero if end of iteration. */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr) +{ + struct entry *remember_e, *remember_parent; + int ret; + + /* Do the removal */ + if (NULL == (itr->parent)) + { + /* element is head of a chain */ + itr->h->table[itr->index] = itr->e->next; + } else { + /* element is mid-chain */ + itr->parent->next = itr->e->next; + } + /* itr->e is now outside the hashtable */ + remember_e = itr->e; + itr->h->entrycount--; + freekey(remember_e->k); + + /* Advance the iterator, correcting the parent */ + remember_parent = itr->parent; + ret = hashtable_iterator_advance(itr); + if (itr->parent == remember_e) { itr->parent = remember_parent; } + free(remember_e); + return ret; +} + +/*****************************************************************************/ +int /* returns zero if not found */ +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k) +{ + struct entry *e, *parent; + unsigned int hashvalue, index; + + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + + e = h->table[index]; + parent = NULL; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + itr->index = index; + itr->e = e; + itr->parent = parent; + itr->h = h; + return -1; + } + parent = e; + e = e->next; + } + return 0; +} + + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable_itr.h b/runtime/hashtable_itr.h new file mode 100644 index 00000000..1c206b6e --- /dev/null +++ b/runtime/hashtable_itr.h @@ -0,0 +1,112 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_ITR_CWC22__ +#define __HASHTABLE_ITR_CWC22__ +#include "hashtable.h" +#include "hashtable_private.h" /* needed to enable inlining */ + +/*****************************************************************************/ +/* This struct is only concrete here to allow the inlining of two of the + * accessor functions. */ +struct hashtable_itr +{ + struct hashtable *h; + struct entry *e; + struct entry *parent; + unsigned int index; +}; + + +/*****************************************************************************/ +/* hashtable_iterator + */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h); + +/*****************************************************************************/ +/* hashtable_iterator_key + * - return the value of the (key,value) pair at the current position */ + +static inline void * +hashtable_iterator_key(struct hashtable_itr *i) +{ + return i->e->k; +} + +/*****************************************************************************/ +/* value - return the value of the (key,value) pair at the current position */ + +static inline void * +hashtable_iterator_value(struct hashtable_itr *i) +{ + return i->e->v; +} + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* remove - remove current element and advance the iterator to the next element + * NB: if you need the value to free it, read it before + * removing. ie: beware memory leaks! + * returns zero if advanced to end of table */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* search - overwrite the supplied iterator, to point to the entry + * matching the supplied key. + h points to the hashtable to be searched. + * returns zero if not found. */ +int +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ +int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ +{ \ + return (hashtable_iterator_search(i,h,k)); \ +} + + + +#endif /* __HASHTABLE_ITR_CWC22__*/ + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable_private.h b/runtime/hashtable_private.h new file mode 100644 index 00000000..10b82da4 --- /dev/null +++ b/runtime/hashtable_private.h @@ -0,0 +1,86 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_PRIVATE_CWC22_H__ +#define __HASHTABLE_PRIVATE_CWC22_H__ + +#include "hashtable.h" + +/*****************************************************************************/ +struct entry +{ + void *k, *v; + unsigned int h; + struct entry *next; +}; + +struct hashtable { + unsigned int tablelength; + struct entry **table; + unsigned int entrycount; + unsigned int loadlimit; + unsigned int primeindex; + unsigned int (*hashfn) (void *k); + int (*eqfn) (void *k1, void *k2); + void (*dest) (void *v); /* destructor for values, if NULL use free() */ +}; + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k); + +/*****************************************************************************/ +/* indexFor */ +static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) { + return (hashvalue % tablelength); +}; + +/* Only works if tablelength == 2^N */ +/*static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) +{ + return (hashvalue & (tablelength - 1u)); +} +*/ + +/*****************************************************************************/ +#define freekey(X) free(X) +/*define freekey(X) ; */ + + +/*****************************************************************************/ + +#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ -- cgit v1.2.3 From 4b9a92bc725f7436b7958e673a9665a90b548e86 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 12:46:39 +0000 Subject: omhdfs: fixed some issues with previous commit --- plugins/omhdfs/omhdfs.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index eefea722..b075432d 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -254,7 +254,6 @@ fileWrite(file_t *pFile, uchar *buf) size_t lenWrite; DEFiRet; - assert(pFile->fh != NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); @@ -391,17 +390,18 @@ BEGINdoHUP file_t *pFile; struct hashtable_itr *itr; CODESTARTdoHUP - /* Iterator constructor only returns a valid iterator if - * the hashtable is not empty */ - itr = hashtable_iterator(files); - if(hashtable_count(files) > 0) - { - do { - pFile = (file_t *) hashtable_iterator_value(itr); - fileClose(pFile); - DBGPRINTF("imuxsock: HUP, closing file %s\n", pFile->name); - } while (hashtable_iterator_advance(itr)); - } + DBGPRINTF("omhdfs: HUP received (file count %d)\n", hashtable_count(files)); + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(files); + if(hashtable_count(files) > 0) + { + do { + pFile = (file_t *) hashtable_iterator_value(itr); + fileClose(pFile); + DBGPRINTF("omhdfs: HUP, closing file %s\n", pFile->name); + } while (hashtable_iterator_advance(itr)); + } ENDdoHUP -- cgit v1.2.3 From bbf38b082a17c77dfa86dcdbd0b7b7b6d36c07b6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 16:17:49 +0200 Subject: imuxsock: fixed dumb mistake of last commit I left a testing setting in, resulting in /dev/log never being deleted... --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 906e28e6..75a6b00b 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -369,7 +369,7 @@ openLogSocket(lstn_t *pLstn) return -1; if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { - bSysSockFromSystemd = 1; /* set default */ + bSysSockFromSystemd = 0; /* set default */ int r; /* System log socket code. Check whether an FD was passed in from systemd. If -- cgit v1.2.3 From a627ac99ba2c3404ca926a19fb06cbd6f43b53c8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 16:25:49 +0200 Subject: omhdfs: added ability to create non-existing directories in name path --- plugins/omhdfs/omhdfs.c | 128 +++++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index b075432d..9705b7fd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -97,54 +97,41 @@ CODESTARTdbgPrintInstInfo ENDdbgPrintInstInfo - -#if 0 -static void prepareFile(instanceData *pData, uchar *newFileName) +/* note that hdfsFileExists() does not work, so we did our + * own function to see if a pathname exists. Returns 0 if the + * file does not exists, something else otherwise. Note that + * we can also check a directroy (if that matters...) + */ +static int +HDFSFileExists(hdfsFS fs, uchar *name) { - if(access((char*)newFileName, F_OK) == 0) { - /* file already exists */ - pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, - pData->fCreateMode); + int r; + hdfsFileInfo *info; + + info = hdfsGetPathInfo(fs, (char*)name); + /* if things go wrong, we assume it is because the file + * does not exist. We do not get too much information... + */ + if(info == NULL) { + r = 0; } else { - pData->fh = -1; - /* file does not exist, create it (and eventually parent directories */ - if(pData->bCreateDirs) { - /* we fist need to create parent dirs if they are missing - * We do not report any errors here ourselfs but let the code - * fall through to error handler below. - */ - if(makeFileParentDirs(newFileName, strlen((char*)newFileName), - pData->fDirCreateMode, pData->dirUID, - pData->dirGID, pData->bFailOnChown) != 0) { - return; /* we give up */ - } - } - /* no matter if we needed to create directories or not, we now try to create - * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) - */ - pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, - pData->fCreateMode); - if(pData->fh != -1) { - /* check and set uid/gid */ - if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { - /* we need to set owner/group */ - if(fchown(pData->fh, pData->fileUID, - pData->fileGID) != 0) { - if(pData->bFailOnChown) { - int eSave = errno; - close(pData->fh); - pData->fh = -1; - errno = eSave; - } - /* we will silently ignore the chown() failure - * if configured to do so. - */ - } - } - } + r = 1; + hdfsFreeFileInfo(info, 1); } + return r; +} + +static inline rsRetVal +HDFSmkdir(hdfsFS fs, uchar *name) +{ + DEFiRet; + if(hdfsCreateDirectory(fs, (char*)name) == -1) + ABORT_FINALIZE(RS_RET_ERR); + +finalize_it: + RETiRet; } -#endif + /* ---BEGIN FILE OBJECT---------------------------------------------------- */ /* This code handles the "file object". This is split from the actual @@ -195,6 +182,42 @@ fileObjDestruct(file_t **ppFile) return RS_RET_OK; } + +/* check, and potentially create, all names inside a path */ +static rsRetVal +filePrepare(file_t *pFile) +{ + uchar *p; + uchar *pszWork; + size_t len; + DEFiRet; + + if(HDFSFileExists(pFile->fs, pFile->name)) + FINALIZE; + + /* file does not exist, create it (and eventually parent directories */ + if(1) { // check if bCreateDirs + len = ustrlen(pFile->name) + 1; + CHKmalloc(pszWork = MALLOC(sizeof(uchar) * len)); + memcpy(pszWork, pFile->name, len); + for(p = pszWork+1 ; *p ; p++) + if(*p == '/') { + /* temporarily terminate string, create dir and go on */ + *p = '\0'; + if(!HDFSFileExists(pFile->fs, pszWork)) { + CHKiRet(HDFSmkdir(pFile->fs, pszWork)); + } + *p = '/'; + } + free(pszWork); + return 0; + } + +finalize_it: + RETiRet; +} + + /* this function is to be used as destructor for the * hash table code. */ @@ -222,6 +245,9 @@ fileOpen(file_t *pFile) DBGPRINTF("omhdfs: error can not connect to hdfs\n"); ABORT_FINALIZE(RS_RET_SUSPENDED); } + + CHKiRet(filePrepare(pFile)); + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_APPEND, 0, 0, 0); if(pFile->fh == NULL) { /* maybe the file does not exist, so we try to create it now. @@ -271,7 +297,8 @@ fileWrite(file_t *pFile, uchar *buf) tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, "omhdfs: failed to write %s, expected %lu bytes, " - "written %lu\n", pFile->name, lenWrite, (unsigned long) num_written_bytes); + "written %lu\n", pFile->name, (unsigned long) lenWrite, + (unsigned long) num_written_bytes); ABORT_FINALIZE(RS_RET_SUSPENDED); } @@ -410,17 +437,6 @@ ENDdoHUP */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { -/* - fileUID = -1; - fileGID = -1; - dirUID = -1; - dirGID = -1; - bFailOnChown = 1; - iDynaFileCacheSize = 10; - fCreateMode = 0644; - fDirCreateMode = 0700; - bCreateDirs = 1; -*/ hdfsHost = NULL; hdfsPort = 0; return RS_RET_OK; -- cgit v1.2.3 From 6ea4a38079cb1508c103930e327fa92ada97176f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 5 Oct 2010 14:06:27 +0200 Subject: updated ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index f9b79392..055f40f4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,8 @@ Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now supports up to 50 different sockets for input - some code cleanup in imuxsock (consider this a release a major modification, especially if problems show up) +- bugfix: /dev/log was unlinked even when passed in from systemd + in which case it should be preserved as systemd owns it --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters -- cgit v1.2.3 From b5afa1a5be2909ed13f77ef0e0804bab75325198 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 5 Oct 2010 15:28:46 +0200 Subject: preparing for 5.7.1 --- ChangeLog | 4 +--- doc/manual.html | 2 +- doc/omhdfs.html | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3bb3d882..f552781a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,6 @@ --------------------------------------------------------------------------- -Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-09-?? +Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-10-05 - support for Hadoop's HDFS added (via omhdfs) ---------------------------------------------------------------------------- -Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log socket itself (thanks to Lennart Poettering for the suggesting this feature) diff --git a/doc/manual.html b/doc/manual.html index 08334c5d..6affe746 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.0 (beta branch) of rsyslog. +

    This documentation is for version 5.7.1 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might diff --git a/doc/omhdfs.html b/doc/omhdfs.html index 3849f167..ef7e60c5 100644 --- a/doc/omhdfs.html +++ b/doc/omhdfs.html @@ -5,7 +5,7 @@

    Unix sockets Output Module (omhdfs)

    Module Name:    omhdfs

    -

    Available since:    5.7.2

    +

    Available since:    5.7.1

    Author: Rainer Gerhards <rgerhards@adiscon.com>

    Description:

    This module supports writing message into files on Hadoop's HDFS -- cgit v1.2.3 From c0de8c6dc4ec4ac270b0c313ea5a1d2689b44db3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 6 Oct 2010 12:37:24 +0200 Subject: bugfix/imuxsock: did not compile on platforms without SCM_CREDENTIALS Note: we do rate-limiting by the pid and we obtain the pid via SCM_CREDENTIALS socket options. So this patch effectively disables the (new) ratelimiting capability. In a later patch, I'll see that I can provide a global ratelimiting capability, which could always be used (an alternative may be using the tag, will check this out as well). --- configure.ac | 5 +++++ plugins/imuxsock/imuxsock.c | 14 +++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 7254b99d..0dbe0f5d 100644 --- a/configure.ac +++ b/configure.ac @@ -110,6 +110,11 @@ AC_FUNC_STRERROR_R AC_FUNC_VPRINTF AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r getline malloc_trim prctl epoll_create epoll_create1 fdatasync]) +# the check below is probably ugly. If someone knows how to do it in a better way, please +# let me know! -- rgerhards, 2010-10-06 +AC_CHECK_DECL([SCM_CREDENTIALS], [AC_DEFINE(HAVE_SCM_CREDENTIALS, [1], [set define])], [], [#include +#include ]) + # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) AC_TRY_COMPILE([ diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 41bff4f9..5b548602 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -406,6 +406,7 @@ openLogSocket(lstn_t *pLstn) CHKiRet(createLogSocket(pLstn)); } +# if HAVE_SCM_CREDENTIALS if(pLstn->bUseCreds) { one = 1; if(setsockopt(pLstn->fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { @@ -417,6 +418,9 @@ openLogSocket(lstn_t *pLstn) pLstn->bUseCreds = 0; } } +# else /* HAVE_SCM_CREDENTIALS */ + pLstn->bUseCreds = 0; +# endif /* HAVE_SCM_CREDENTIALS */ finalize_it: if(iRet != RS_RET_OK) { @@ -513,7 +517,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) rs_ratelimit_state_t *ratelimiter = NULL; DEFiRet; -// TODO: handle format errors?? + /* TODO: handle format errors?? */ /* we need to parse the pri first, because we need the severity for * rate-limiting as well. */ @@ -530,8 +534,10 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) facil = LOG_FAC(pri); sever = LOG_PRI(pri); - if(sever >= pLstn->ratelimitSev) + if(sever >= pLstn->ratelimitSev) { + /* note: if cred == NULL, then ratelimiter == NULL as well! */ findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ + } datetime.getCurrTime(&st, &tt); if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt, cred->pid)) { @@ -637,6 +643,7 @@ static rsRetVal readSocket(lstn_t *pLstn) dbgprintf("Message from UNIX socket: #%d\n", pLstn->fd); if(iRcvd > 0) { cred = NULL; +# if HAVE_SCM_CREDENTIALS if(pLstn->bUseCreds) { dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { @@ -644,11 +651,12 @@ static rsRetVal readSocket(lstn_t *pLstn) if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { cred = (struct ucred*) CMSG_DATA(cm); dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); - //break; + break; } } dbgprintf("XXX: post CM loop\n"); } +# endif /* HAVE_SCM_CREDENTIALS */ CHKiRet(SubmitMsg(pRcv, iRcvd, pLstn, cred)); } else if(iRcvd < 0 && errno != EINTR) { char errStr[1024]; -- cgit v1.2.3 From d51d6847a84eeec573ce2dd4feeee991d3eb0bcc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 6 Oct 2010 12:54:33 +0200 Subject: previous bugfix was incomplete, resolving this --- plugins/imuxsock/imuxsock.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 5b548602..566dde1b 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -66,6 +66,10 @@ MODULE_TYPE_INPUT #endif #endif +/* emulate struct ucred for platforms that do not have it */ +#ifndef HAVE_SCM_CREDENTIALS +struct ucred { int pid; }; +#endif /* handle some defines missing on more than one platform */ #ifndef SUN_LEN -- cgit v1.2.3 From e6b184ad84e055c7272c41706c2fd8f86f6ac056 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 7 Oct 2010 13:53:07 +0200 Subject: doc: documented way to turn off imuxsock ratelimiting --- doc/imuxsock.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 1ab99a76..4dd28f26 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -39,6 +39,7 @@ config statements just use the prefix $IMUXSockRateLimit... but otherwise works exactly the same. When working with severities, please keep in mind that higher severity numbers mean lower severity and configure things accordingly. +To turn off rate limiting, set the interval to zero.

    Unix log sockets can be flow-controlled. That is, if processing queues fill up, the unix socket reader is blocked for a short while. This may be useful to prevent overruning the queues (which may cause exessive disk-io where it actually would not be needed). However, @@ -55,7 +56,7 @@ the implications. Note that for many systems, turning on flow control does not h

  • $InputUnixListenSocketFlowControl [on/off] - specifies if flow control should be applied to the next socket.
  • $IMUXSockRateLimitInterval [number] - specifies the rate-limiting -interval in seconds. Default value is 5 seconds. +interval in seconds. Default value is 5 seconds. Set it to 0 to turn rate limiting off.
  • $IMUXSockRateLimitBurst [number] - specifies the rate-limiting burst in number of messages. Default is 200. @@ -78,7 +79,7 @@ be obtained from the log socket itself. If so, the TAG part of the message is re It is recommended to turn this option on, but the default is "off" to keep compatible with earlier versions of rsyslog. This option was introduced in 5.7.0.
  • $SystemLogRateLimitInterval [number] - specifies the rate-limiting -interval in seconds. Default value is 5 seconds. +interval in seconds. Default value is 5 seconds. Set it to 0 to turn rate limiting off.
  • $SystemLogRateLimitBurst [number] - specifies the rate-limiting burst in number of messages. Default is 200. -- cgit v1.2.3 From ed324a9a610f91f54514eb713ff5593bde1012e6 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Tue, 5 Oct 2010 17:41:09 +0200 Subject: Enable rsyslog.socket together with rsyslog.service When the rsyslog service is enabled via "systemctl enable rsyslog.service" also enable the socket unit. --- rsyslog.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/rsyslog.service.in b/rsyslog.service.in index ea966cc3..b3c55515 100644 --- a/rsyslog.service.in +++ b/rsyslog.service.in @@ -6,3 +6,4 @@ ExecStart=@sbindir@/rsyslogd -n -c5 [Install] WantedBy=multi-user.target +Also=rsyslog.socket -- cgit v1.2.3 From 8c3d40b798e3ed68fb03629d87b55140cb6bc044 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 Oct 2010 17:11:00 +0200 Subject: removed need for math library by doing math a little bit more optimal in hash table code. Also reduced memory requirement for imuxsock hash tables (expected number of connections was set too high -- table can be extended dynamically). --- doc/imuxsock.html | 6 ++++++ plugins/imuxsock/imuxsock.c | 4 ++-- runtime/hashtable.c | 16 +++++++++++++--- tools/Makefile.am | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 4dd28f26..ee5db22d 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -139,6 +139,12 @@ the $InputUnixListenSocketCreatePath and the $InputUnixListenSocketHostName.

    $InputUnixListenSocketCreatePath on # turn on for *next* socket $InputUnixListenSocketHostName /var/run/sshd/dev/log +

    The following sample is used to turn off input rate limiting on the system log +socket. +

    [rsyslog.conf overview] [manual index] [rsyslog site]

    This documentation is part of the diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 566dde1b..0eee1122 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -283,7 +283,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); if(ratelimitInterval > 0) { - if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { + if((listeners[nfd].ht = create_hashtable(100, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); @@ -761,7 +761,7 @@ CODESTARTwillRun if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; if(ratelimitIntervalSysSock > 0) { - if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { + if((listeners[0].ht = create_hashtable(100, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); diff --git a/runtime/hashtable.c b/runtime/hashtable.c index 41fc60fe..a01fa7d9 100644 --- a/runtime/hashtable.c +++ b/runtime/hashtable.c @@ -23,7 +23,17 @@ static const unsigned int primes[] = { 805306457, 1610612741 }; const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); -const float max_load_factor = 0.65; + +#define MAX_LOAD_FACTOR 65 /* to get real factor, divide by 100! */ + +/* compute max load. We use a constant factor of 0.65, but do + * everything times 100, so that we do not need floats. + */ +static inline unsigned +getLoadLimit(unsigned size) +{ + return (unsigned int) ((unsigned long long) size * MAX_LOAD_FACTOR) / 100; +} /*****************************************************************************/ struct hashtable * @@ -50,7 +60,7 @@ create_hashtable(unsigned int minsize, h->hashfn = hashf; h->eqfn = eqf; h->dest = dest; - h->loadlimit = (unsigned int) ceil(size * max_load_factor); + h->loadlimit = getLoadLimit(size); return h; } @@ -123,7 +133,7 @@ hashtable_expand(struct hashtable *h) } } h->tablelength = newsize; - h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + h->loadlimit = getLoadLimit(newsize); return -1; } diff --git a/tools/Makefile.am b/tools/Makefile.am index 6541194a..96657ad4 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -36,7 +36,7 @@ rsyslogd_SOURCES = \ \ ../dirty.h rsyslogd_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) -lm +rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS -- cgit v1.2.3 From ed9fd3541f07929008a64f5b6ea3bcce70e62c25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 15 Oct 2010 08:23:29 +0200 Subject: imfile: bug fixes either one or two bugs fixed ;) Definitely a problem where no state file is written when working with relative pathes. Also, some problems with offsets should be fixed for very large files. However, I could not yet experimentally show the issue so it probably needs more verification. --- ChangeLog | 4 ++++ plugins/imfile/imfile.c | 5 ++++- runtime/stream.c | 6 +++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index f552781a..fd0e0487 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,8 @@ --------------------------------------------------------------------------- +Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-10-05 +- bugfix: imfile state file was not written when relative file name + for it was specified +--------------------------------------------------------------------------- Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-10-05 - support for Hadoop's HDFS added (via omhdfs) - imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 8a10e26f..e067014e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -337,12 +337,15 @@ persistStrmState(fileInfo_t *pInfo) { DEFiRet; strm_t *psSF = NULL; /* state file (stream) */ + size_t lenDir; ASSERT(pInfo != NULL); /* TODO: create a function persistObj in obj.c? */ CHKiRet(strm.Construct(&psSF)); - CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + lenDir = ustrlen(glbl.GetWorkDir()); + if(lenDir > 0) + CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), lenDir)); CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); diff --git a/runtime/stream.c b/runtime/stream.c index b4295762..7f965029 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1473,7 +1473,7 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) { DEFiRet; int i; - long l; + int64 ll; ISOBJ_TYPE_assert(pThis, strm); ISOBJ_TYPE_assert(pStrm, strm); @@ -1495,8 +1495,8 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) i = pThis->tOpenMode; objSerializeSCALAR_VAR(pStrm, tOpenMode, INT, i); - l = (long) pThis->iCurrOffs; - objSerializeSCALAR_VAR(pStrm, iCurrOffs, LONG, l); + ll = pThis->iCurrOffs; + objSerializeSCALAR_VAR(pStrm, iCurrOffs, INT64, ll); CHKiRet(obj.EndSerialize(pStrm)); -- cgit v1.2.3 From ffd08f2a6caaaddb86ccbec4206bf560d34fcfd7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 24 Oct 2010 14:31:12 +0200 Subject: imfile improvements - added the $InputFilePersistStateInterval config directive to imfile - changed imfile so that the state file is never deleted (makes imfile more robust in regard to fatal failures) --- ChangeLog | 3 +++ doc/imfile.html | 10 ++++++++++ plugins/imfile/imfile.c | 35 +++++++++++++++++------------------ 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/ChangeLog b/ChangeLog index a4ac46b9..acb0ef5e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,9 @@ Version 4.7.3 [v4-devel] (rgerhards), 2010-??-?? - bugfix: imfile utilizes 32 bit to track offset. Most importantly, this problem can not experienced on Fedora 64 bit OS (which has 64 bit long's!) +- added the $InputFilePersistStateInterval config directive to imfile +- changed imfile so that the state file is never deleted (makes imfile + more robust in regard to fatal failures) --------------------------------------------------------------------------- Version 4.7.2 [v4-devel] (rgerhards), 2010-05-03 - bugfix: problems with atomic operations emulaton diff --git a/doc/imfile.html b/doc/imfile.html index af0413dd..3687302b 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -86,6 +86,16 @@ level may be needed. Even if you need quick response, 1 seconds should be well enough. Please note that imfile keeps reading files as long as there is any data in them. So a "polling sleep" will only happen when nothing is left to be processed.

    • +
  • $InputFilePollInterval [lines]
    +Available in 4.7.3+
    +Specifies how often the state file shall be written when processing the input +file. The default value is 0, which means a new state file is only written when +the monitored files is being closed (end of rsyslogd execution). Any other +value n means that the state file is written every time n file lines have +been processed. This setting can be used to guard against message duplication due +to fatal errors (like power fail). Note that this setting affects imfile +performance, especially when set to a low value. Frequently writing the state +file is very time consuming. Caveats/Known Bugs:

    So far, only 100 files can be monitored. If more are needed, diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 3981f9f7..91493bb1 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -67,15 +67,21 @@ typedef struct fileInfo_s { uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; + int nRecords; /**< How many records did we process before persisting the stream? */ + int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ } fileInfo_t; +/* forward definitions */ +static rsRetVal persistStrmState(fileInfo_t *pInfo); + /* config variables */ static uchar *pszFileName = NULL; static uchar *pszFileTag = NULL; static uchar *pszStateFile = NULL; static int iPollInterval = 10; /* number of seconds to sleep when there was no file activity */ +static int iPersistStateInterval = 0; /* how often if state file to be persisted? (default 0->never) */ static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ @@ -154,10 +160,9 @@ openFile(fileInfo_t *pThis) CHKiRet(strm.SeekCurrOffs(pThis->pStrm)); - /* OK, we could successfully read the file, so we now can request that it be deleted. - * If we need it again, it will be written on the next shutdown. + /* note: we do not delete the state file, so that the last position remains + * known even in the case that rsyslogd aborts for some reason (like powerfail) */ - psSF->bDeleteOnClose = 1; finalize_it: if(psSF != NULL) @@ -211,6 +216,10 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ + if(pThis->iPersistStateInterval > 0 && pThis->nRecords++ >= pThis->iPersistStateInterval) { + persistStrmState(pThis); + pThis->nRecords = 0; + } } finalize_it: @@ -244,27 +253,12 @@ finalize_it: * IMPORTANT: the calling interface of this function can NOT be modified. It actually is * called by pthreads. The provided argument is currently not being used. */ -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ static void inputModuleCleanup(void __attribute__((unused)) *arg) { BEGINfunc -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ - - - - /* so far not needed */ - - - -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ ENDfunc } -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ /* This function is called by the framework to gather the input. The module stays @@ -499,6 +493,9 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->iSeverity = iSeverity; pThis->iFacility = iFacility; + pThis->iPersistStateInterval = iPersistStateInterval; + pThis->nRecords = 0; + iPersistStateInterval = 0; } else { errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one"); ABORT_FINALIZE(RS_RET_OUT_OF_DESRIPTORS); @@ -544,6 +541,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepollinterval", 0, eCmdHdlrInt, NULL, &iPollInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, + NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); /* that command ads a new file! */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrunfilemonitor", 0, eCmdHdlrGetWord, addMonitor, NULL, STD_LOADABLE_MODULE_ID)); -- cgit v1.2.3 From ab6e674b0bae88d3a91a30f4e32fbb857096964f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Oct 2010 16:32:33 +0200 Subject: doc/imfile: fixed small but important typo --- doc/imfile.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/imfile.html b/doc/imfile.html index 3687302b..89be3292 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -86,7 +86,7 @@ level may be needed. Even if you need quick response, 1 seconds should be well enough. Please note that imfile keeps reading files as long as there is any data in them. So a "polling sleep" will only happen when nothing is left to be processed.

    • -
  • $InputFilePollInterval [lines]
    +
  • $InputFilePersistStateInterval [lines]
    Available in 4.7.3+
    Specifies how often the state file shall be written when processing the input file. The default value is 0, which means a new state file is only written when -- cgit v1.2.3 From 9e6ab1494d95da61095867db209674975b1b1a2b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Nov 2010 13:56:53 +0100 Subject: preparing for 4.7.3 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0fe62c5f..a266aafd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.7.3 [v4-devel] (rgerhards), 2010-??-?? +Version 4.7.3 [v4-devel] (rgerhards), 2010-11-25 - bugfix(important): problem in TLS handling could cause rsyslog to loop in a tight loop, effectively disabling functionality and bearing the risk of unresponsiveness of the whole system. diff --git a/configure.ac b/configure.ac index f265727f..70ec65d3 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.7.2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.7.3],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index 0e4166d0..a95e8eec 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 4.7.2 (v4-devel branch) of rsyslog. +

    This documentation is for version 4.7.3 (v4-devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 91cf297043e20d2dae8b00c20efadcc388357a86 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Nov 2010 14:29:02 +0100 Subject: added forgotten testcase files --- tests/manyptcp.sh | 13 +++++++++++++ tests/testsuites/manyptcp.conf | 12 ++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 tests/manyptcp.sh create mode 100644 tests/testsuites/manyptcp.conf diff --git a/tests/manyptcp.sh b/tests/manyptcp.sh new file mode 100755 index 00000000..3ed5493b --- /dev/null +++ b/tests/manyptcp.sh @@ -0,0 +1,13 @@ +# test many concurrent tcp connections +echo ==================================================================================== +echo TEST: \[manyptcp.sh\]: test imptcp with large connection count +source $srcdir/diag.sh init +source $srcdir/diag.sh startup manyptcp.conf +# the config file specifies exactly 1100 connections +source $srcdir/diag.sh tcpflood -c1000 -m40000 +# the sleep below is needed to prevent too-early termination of the tcp listener +sleep 1 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/manyptcp.conf b/tests/testsuites/manyptcp.conf new file mode 100644 index 00000000..4069f977 --- /dev/null +++ b/tests/testsuites/manyptcp.conf @@ -0,0 +1,12 @@ +# Test for tcp "flood" testing +# rgerhards, 2009-04-08 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imptcp/.libs/imptcp +$MainMsgQueueTimeoutShutdown 10000 +$MaxOpenFiles 2000 +$InputPTCPServerRun 13514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit v1.2.3 From b9151f6a1708b2fb7e9518e73fcf42d86b0364a9 Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Thu, 25 Nov 2010 15:44:49 +0100 Subject: bugfix: atomic increment for msg object may not work correct on all platforms. Signed-off-by: Rainer Gerhards --- ChangeLog | 4 ++++ runtime/msg.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index a266aafd..bbf900fa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,8 @@ --------------------------------------------------------------------------- +Version 4.7.4 [v4-???] (rgerhards), 2010-11-?? +- bugfix: atomic increment for msg object may not work correct on all + platforms. Thanks to Chris Metcalf for the patch +--------------------------------------------------------------------------- Version 4.7.3 [v4-devel] (rgerhards), 2010-11-25 - bugfix(important): problem in TLS handling could cause rsyslog to loop in a tight loop, effectively disabling functionality and bearing the diff --git a/runtime/msg.h b/runtime/msg.h index cda206fc..a6923d52 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -60,9 +60,9 @@ struct msg { flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ pthread_mutex_t mut; + int iRefCount; /* reference counter (0 = unused) */ bool bDoLock; /* use the mutex? */ bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ - short iRefCount; /* reference counter (0 = unused) */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, * it is available when we have a forwarder (e.g. rfc3195d) using local -- cgit v1.2.3 From 1aac36dfd077073a76b80368f893810e1741a1ec Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 30 Nov 2010 11:23:12 +0100 Subject: bumped version to 5.7.2 --- ChangeLog | 6 +++++- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 55f375e3..e18c152e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- -Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-10-05 +Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-11-26 +- bugfix(important): problem in TLS handling could cause rsyslog to loop + in a tight loop, effectively disabling functionality and bearing the + risk of unresponsiveness of the whole system. + Bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=194 - bugfix: imfile state file was not written when relative file name for it was specified - bugfix: compile failed on systems without epoll_create1() diff --git a/configure.ac b/configure.ac index c2c7e8be..c1b41115 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 6affe746..7fdf3f14 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.1 (beta branch) of rsyslog. +

    This documentation is for version 5.7.2 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 0b18c17b2850152203ce9db648ce06212ab67157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Tue, 30 Nov 2010 13:04:58 +0100 Subject: Fix a potential missing '\0' on too long strings. By implementing a trivial strlcpy it's much easier to detect string truncations and react to them. This also gives a noticeable speedup in buffer handling (can be HUGE), since strlcpy() doesn't clear all the buffer entry before writing data. Converted all uses of strncpy() into strlcpy(). Also, we don't need to check for some null pointers, as there are no malloc-like operations in the doAction loop. --- plugins/omoracle/omoracle.c | 19 +++++++++++++---- tests/cfg4.testin | 52 +++++++-------------------------------------- 2 files changed, 23 insertions(+), 48 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 48ee1fa4..30b5834b 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -127,6 +127,13 @@ typedef struct _instanceData { struct oracle_batch batch; } instanceData; +/* To be honest, strlcpy is faster than strncpy and makes very easy to + * detect if a message has been truncated. */ +#ifndef strlcpy +#define strlcpy(dst,src,sz) snprintf((dst), (sz), "%s", (src)) +#endif + + /** Database name, to be filled by the $OmoracleDB directive */ static char* db_name; /** Database user name, to be filled by the $OmoracleDBUser @@ -529,7 +536,7 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction - int i; + int i, sz; char **params = (char**) ppString[0]; CODESTARTdoAction @@ -540,9 +547,13 @@ CODESTARTdoAction for (i = 0; i < pData->batch.arguments && params[i]; i++) { dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); - strncpy(pData->batch.parameters[i][pData->batch.n], params[i], - pData->batch.param_size); - CHKmalloc(pData->batch.parameters[i][pData->batch.n]); + sz = strlcpy(pData->batch.parameters[i][pData->batch.n], + params[i], pData->batch.param_size); + if (sz >= pData->batch.param_size) + errmsg.LogError(0, NO_ERRCODE, + "Possibly truncated %d column of '%s' " + "statement: %s", i, + pData->txt_statement, params[i]); } pData->batch.n++; diff --git a/tests/cfg4.testin b/tests/cfg4.testin index a49c0fb6..2dc0e830 100644 --- a/tests/cfg4.testin +++ b/tests/cfg4.testin @@ -12,48 +12,6 @@ # If you do not load inputs, nothing happens! # You may need to set the module load path if modules are not found. -#$ModLoad immark # provides --MARK-- message capability -#$ModLoad imuxsock # provides support for local system logging (e.g. via logger command) -#$ModLoad imklog # kernel logging (formerly provided by rklogd) - -# Log all kernel messages to the console. -# Logging much else clutters up the screen. -#kern.* /dev/console - -# Log anything (except mail) of level info or higher. -# Don't log private authentication messages! -*.info;mail.none;authpriv.none;cron.none -/var/log/messages - -# The authpriv file has restricted access. -authpriv.* /var/log/secure - -# Log all the mail messages in one place. -mail.* -/var/log/maillog - - -# Log cron stuff -cron.* -/var/log/cron - -# Everybody gets emergency messages -*.emerg * - -# Save news errors of level crit and higher in a special file. -uucp,news.crit -/var/log/spooler - -# Save boot messages also to boot.log -local7.* /var/log/boot.log - -# Remote Logging (we use TCP for reliable delivery) -# An on-disk queue is created for this action. If the remote host is -# down, messages are spooled to disk and sent when it is up again. -#$WorkDirectory /rsyslog/spool # where to place spool files -#$ActionQueueFileName uniqName # unique name prefix for spool files -#$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) -#$ActionQueueSaveOnShutdown on # save messages to disk on shutdown -#$ActionQueueType LinkedList # run asynchronously -#$ActionResumeRetryCount -1 # infinite retries if host is down -# remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional -#*.* @@remote-host:514 # ######### Receiving Messages from Remote Hosts ########## @@ -63,5 +21,11 @@ local7.* /var/log/boot.log #$InputTCPServerRun 514 # start up TCP listener at port 514 # UDP Syslog Server: -#$ModLoad imudp.so # provides UDP syslog reception -#$UDPServerRun 514 # start a UDP syslog server at standard port 514 +$ModLoad imudp.so # provides UDP syslog reception +$ModLoad omoracle.so +$UDPServerRun 514 # start a UDP syslog server at standard port 514 + +$IncludeConfig /home/munoz/logging/rsyslog/20*conf +$IncludeConfig /home/munoz/logging/rsyslog/30*conf + +#*.* ~ -- cgit v1.2.3 From c4c20c18aae7c95c1e69c10d2ea9efbc3c2c1913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Tue, 30 Nov 2010 13:04:59 +0100 Subject: Fix the build system to build outisde CERN. It should build against Oracle 11, too. Depending on the user's installation, either a single ORACLE_HOME environment variable or ORACLE_LIB_PATH and ORACLE_INCLUDE_PATH environment variables will be needed. --- configure.ac | 52 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/configure.ac b/configure.ac index 70ec65d3..b5c2d1c2 100644 --- a/configure.ac +++ b/configure.ac @@ -487,7 +487,7 @@ AC_SUBST(PGSQL_LIBS) # oracle (OCI) support AC_ARG_ENABLE(oracle, - [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@])], + [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@]. (Check your ORACLE_HOME environment variable!))], [case "${enableval}" in yes) enable_oracle="yes" ;; no) enable_oracle="no" ;; @@ -496,23 +496,41 @@ AC_ARG_ENABLE(oracle, [enable_oracle=no] ) if test "x$enable_oracle" = "xyes"; then - AC_CHECK_PROG( - [HAVE_ORACLE_CONFIG], - [oracle-instantclient-config], - [yes],,, - ) - if test "x${HAVE_ORACLE_CONFIG}" != "xyes"; then - AC_MSG_FAILURE([oracle-instantclient-config not found in PATH]) + if test -d "$ORACLE_HOME" + then + AC_CHECK_LIB([occi], [OCIEnvCreate], + [ORACLE_CFLAGS=-I$ORACLE_HOME/rdbms/public] + [ORACLE_LIBS=-L$ORACLE_HOME/lib -locci], + [AC_MSG_FAILURE([Oracle (OCI) library is missing: wrong oracle home])], + [-I$ORACLE_HOME/rdbms/public/ -L$ORACLE_HOME/lib -locci -lclntsh ] + ) + elif test -d "$ORACLE_LIB_PATH" -a -d "$ORACLE_INCLUDE_PATH" + then + AC_CHECK_LIB([occi], [OCIEnvCreate], + [ORACLE_CFLAGS=-I$ORACLE_INCLUDE_PATH] + [ORACLE_LIBS=-L$ORACLE_LIB_PATH -locci], + [AC_MSG_FAILURE([Oracle (OCI) library is missing: wrong oracle directories])], + [-I$ORACLE_INCLUDE_PATH -L$ORACLE_LIB_PATH -locci -lclntsh ] + ) + else + AC_CHECK_PROG( + [HAVE_ORACLE_CONFIG], + [oracle-instantclient-config], + [yes],,, + ) + if test "x${HAVE_ORACLE_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([oracle-instantclient-config not found in PATH]) + fi + AC_CHECK_LIB( + [occi], + [OCIEnvCreate], + [ORACLE_CFLAGS="`oracle-instantclient-config --cflags`" + ORACLE_LIBS="`oracle-instantclient-config --libs`" + ], + [AC_MSG_FAILURE([Oracle (OCI) libraray is missing])], + [`oracle-instantclient-config --libs --cflags`] + ) fi - AC_CHECK_LIB( - [occi], - [OCIEnvCreate], - [ORACLE_CFLAGS="`oracle-instantclient-config --cflags`" - ORACLE_LIBS="`oracle-instantclient-config --libs`" - ], - [AC_MSG_FAILURE([Oracle (OCI) libraray is missing])], - [`oracle-instantclient-config --libs --cflags`] - ) fi AM_CONDITIONAL(ENABLE_ORACLE, test x$enable_oracle = xyes) AC_SUBST(ORACLE_CFLAGS) -- cgit v1.2.3 From f8769ca19da2eff26dd80ca3b6217d078db1a59f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Dec 2010 18:13:40 +0100 Subject: bugfix: fixed build problems on some platforms namely those that have 32bit atomic operations but not 64 bit ones --- ChangeLog | 4 ++++ runtime/atomic.h | 27 +++++++++++++++++++++++++++ runtime/statsobj.h | 8 ++++---- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index e18c152e..d0661256 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,8 @@ --------------------------------------------------------------------------- +Version 5.7.3 [V5-DEVEL] (rgerhards), 2010-12-?? +- bugfix: fixed build problems on some platforms + namely those that have 32bit atomic operations but not 64 bit ones +--------------------------------------------------------------------------- Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-11-26 - bugfix(important): problem in TLS handling could cause rsyslog to loop in a tight loop, effectively disabling functionality and bearing the diff --git a/runtime/atomic.h b/runtime/atomic.h index 790cb1c6..c022035c 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -208,4 +208,31 @@ #endif +/* we need to handle 64bit atomics seperately as some platforms have + * 32 bit atomics, but not 64 biot ones... -- rgerhards, 2010-12-01 + */ +#ifdef HAVE_ATOMIC_BUILTINS_64BIT +# define ATOMIC_INC_uint64(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) +# define ATOMIC_DEC_unit64(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) + +# define DEF_ATOMIC_HELPER_MUT64(x) +# define INIT_ATOMIC_HELPER_MUT64(x) +# define DESTROY_ATOMIC_HELPER_MUT64(x) +#else +# define ATOMIC_INC_uint64(data, phlpmut) { \ + pthread_mutex_lock(phlpmut); \ + ++(*(data)); \ + pthread_mutex_unlock(phlpmut); \ + } +# define ATOMIC_DEC_uint64(data, phlpmut) { \ + pthread_mutex_lock(phlpmut); \ + --(*(data)); \ + pthread_mutex_unlock(phlpmut); \ + } + +# define DEF_ATOMIC_HELPER_MUT64(x) pthread_mutex_t x +# define INIT_ATOMIC_HELPER_MUT64(x) pthread_mutex_init(&(x), NULL) +# define DESTROY_ATOMIC_HELPER_MUT64(x) pthread_mutex_destroy(&(x)) +#endif /* #ifdef HAVE_ATOMIC_BUILTINS_64BIT */ + #endif /* #ifndef INCLUDED_ATOMIC_H */ diff --git a/runtime/statsobj.h b/runtime/statsobj.h index 7447cded..44c26bea 100644 --- a/runtime/statsobj.h +++ b/runtime/statsobj.h @@ -99,19 +99,19 @@ PROTOTYPEObj(statsobj); */ #define STATSCOUNTER_DEF(ctr, mut) \ intctr_t ctr; \ - DEF_ATOMIC_HELPER_MUT(mut); + DEF_ATOMIC_HELPER_MUT64(mut); #define STATSCOUNTER_INIT(ctr, mut) \ - INIT_ATOMIC_HELPER_MUT(mut); \ + INIT_ATOMIC_HELPER_MUT64(mut); \ ctr = 0; #define STATSCOUNTER_INC(ctr, mut) \ if(GatherStats) \ - ATOMIC_INC(&ctr, &mut); + ATOMIC_INC_uint64(&ctr, &mut); #define STATSCOUNTER_DEC(ctr, mut) \ if(GatherStats) \ - ATOMIC_DEC(&ctr, mut); + ATOMIC_DEC_uint64(&ctr, mut); /* the next macro works only if the variable is already guarded * by mutex (or the users risks a wrong result). It is assumed -- cgit v1.2.3 From 1bfb97e576d779a709e1e2979eef4697b9b0cd16 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Dec 2010 07:28:04 +0100 Subject: bugfix: one type of 64bit atomics was enabled when 32bit atomics were supported also cleaned up some minor things --- runtime/atomic.h | 27 ++++++++++----------------- tools/syslogd.c | 14 +++++++------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/runtime/atomic.h b/runtime/atomic.h index c022035c..3e2c0d18 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -46,7 +46,6 @@ # define ATOMIC_INC(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) # define ATOMIC_INC_AND_FETCH_int(data, phlpmut) __sync_fetch_and_add(data, 1) # define ATOMIC_INC_AND_FETCH_unsigned(data, phlpmut) __sync_fetch_and_add(data, 1) -# define ATOMIC_INC_AND_FETCH_uint64(data, phlpmut) __sync_fetch_and_add(data, 1) # define ATOMIC_DEC(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) # define ATOMIC_DEC_AND_FETCH(data, phlpmut) __sync_sub_and_fetch(data, 1) # define ATOMIC_FETCH_32BIT(data, phlpmut) ((unsigned) __sync_fetch_and_and(data, 0xffffffff)) @@ -160,15 +159,6 @@ return(val); } - static inline unsigned - ATOMIC_INC_AND_FETCH_uint64(uint64 *data, pthread_mutex_t *phlpmut) { - uint64 val; - pthread_mutex_lock(phlpmut); - val = ++(*data); - pthread_mutex_unlock(phlpmut); - return(val); - } - static inline int ATOMIC_DEC_AND_FETCH(int *data, pthread_mutex_t *phlpmut) { int val; @@ -193,13 +183,6 @@ (*data) -= val; pthread_mutex_unlock(phlpmut); } -#if 0 -# warning "atomic builtins not available, using nul operations - rsyslogd will probably be racy!" -# define ATOMIC_INC_AND_FETCH_int(data) (++(data)) -# define ATOMIC_INC_AND_FETCH_unsigned(data) (++(data)) -# define ATOMIC_INC_AND_FETCH_uint64(data) (++(data)) -# define ATOMIC_STORE_1_TO_32BIT(data) (data) = 1 // TODO: del -#endif # define DEF_ATOMIC_HELPER_MUT(x) pthread_mutex_t x # define INIT_ATOMIC_HELPER_MUT(x) pthread_mutex_init(&(x), NULL) # define DESTROY_ATOMIC_HELPER_MUT(x) pthread_mutex_destroy(&(x)) @@ -214,6 +197,7 @@ #ifdef HAVE_ATOMIC_BUILTINS_64BIT # define ATOMIC_INC_uint64(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) # define ATOMIC_DEC_unit64(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) +# define ATOMIC_INC_AND_FETCH_uint64(data, phlpmut) __sync_fetch_and_add(data, 1) # define DEF_ATOMIC_HELPER_MUT64(x) # define INIT_ATOMIC_HELPER_MUT64(x) @@ -230,6 +214,15 @@ pthread_mutex_unlock(phlpmut); \ } + static inline unsigned + ATOMIC_INC_AND_FETCH_uint64(uint64 *data, pthread_mutex_t *phlpmut) { + uint64 val; + pthread_mutex_lock(phlpmut); + val = ++(*data); + pthread_mutex_unlock(phlpmut); + return(val); + } + # define DEF_ATOMIC_HELPER_MUT64(x) pthread_mutex_t x # define INIT_ATOMIC_HELPER_MUT64(x) pthread_mutex_init(&(x), NULL) # define DESTROY_ATOMIC_HELPER_MUT64(x) pthread_mutex_destroy(&(x)) diff --git a/tools/syslogd.c b/tools/syslogd.c index 4964f8fc..70d4cf2e 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2111,11 +2111,6 @@ static void printVersion(void) #else printf("\tFEATURE_LARGEFILE:\t\t\tNo\n"); #endif -#ifdef USE_NETZIP - printf("\tFEATURE_NETZIP (message compression):\tYes\n"); -#else - printf("\tFEATURE_NETZIP (message compression):\tNo\n"); -#endif #if defined(SYSLOG_INET) && defined(USE_GSSAPI) printf("\tGSSAPI Kerberos 5 support:\t\tYes\n"); #else @@ -2127,9 +2122,14 @@ static void printVersion(void) printf("\tFEATURE_DEBUG (debug build, slow code):\tNo\n"); #endif #ifdef HAVE_ATOMIC_BUILTINS - printf("\tAtomic operations supported:\t\tYes\n"); + printf("\t32bit Atomic operations supported:\tYes\n"); +#else + printf("\t32bit Atomic operations supported:\tNo\n"); +#endif +#ifdef HAVE_ATOMIC_BUILTINS64 + printf("\t64bit Atomic operations supported:\tYes\n"); #else - printf("\tAtomic operations supported:\t\tNo\n"); + printf("\t64bit Atomic operations supported:\tNo\n"); #endif #ifdef RTINST printf("\tRuntime Instrumentation (slow code):\tYes\n"); -- cgit v1.2.3 From da75472096c45dd136d41c6e9ad7bf740069d3a1 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Fri, 3 Dec 2010 00:55:48 +0100 Subject: systemd: add ExecReload command (using kill -HUP) http://0pointer.de/public/systemd-man/systemd.service.html --- rsyslog.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/rsyslog.service.in b/rsyslog.service.in index b3c55515..2bcde528 100644 --- a/rsyslog.service.in +++ b/rsyslog.service.in @@ -3,6 +3,7 @@ Description=System Logging Service [Service] ExecStart=@sbindir@/rsyslogd -n -c5 +ExecReload=/bin/kill -HUP $MAINPID [Install] WantedBy=multi-user.target -- cgit v1.2.3 From 06bad730521dc476a7eabbbedf624a4cfbf65b18 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Dec 2010 13:40:23 +0100 Subject: cleanup of cosmetic nit (result of clang static code analyser run) --- tools/syslogd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index fb1c6e0e..5f8d45a8 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3290,7 +3290,7 @@ int realMain(int argc, char **argv) } } - if ((argc -= optind)) + if(argc - optind) usage(); DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s', cwd:%s\n", -- cgit v1.2.3 From 699d0d933ab64941d40df17c69b2c377231924cf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Dec 2010 15:29:20 +0100 Subject: added $LocalHostName config directive & some bugfixing - added $LocalHostName config directive - bugfix: local hostname was pulled too-early, so that some config directives (namely FQDN settings) did not have any effect --- ChangeLog | 3 +++ doc/rsyslog_conf_global.html | 6 ++++++ runtime/cfsysline.c | 3 --- runtime/glbl.c | 27 +++++++++++++++++++-------- tests/testsuites/parse1.conf | 1 + tools/syslogd.c | 6 +++--- 6 files changed, 32 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0982b77a..caa53b8c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,8 @@ --------------------------------------------------------------------------- Version 4.7.4 [v4-???] (rgerhards), 2010-11-?? +- added $LocalHostName config directive +- bugfix: local hostname was pulled too-early, so that some config + directives (namely FQDN settings) did not have any effect - bugfix: atomic increment for msg object may not work correct on all platforms. Thanks to Chris Metcalf for the patch --------------------------------------------------------------------------- diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 8c1cc9a7..b58ae9c2 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -150,6 +150,12 @@ Usually that should not be a big issue, as the restart-type HUP can easily be re something along the lines of "/etc/init.d/rsyslog restart".

  • $IncludeConfig
  • MainMsgQueueCheckpointInterval <number>
    • +
  • $LocalHostName [name] - this directive permits to overwrite the system +hostname with the one specified in the directive. If the directive is given +multiple times, all but the last one will be ignored. Please note that startup +error messages may be issued with the real hostname. This is by design and not +a bug (but one may argue if the design should be changed ;)). Available since +4.7.4+, 5.7.3+, 6.1.3+.
  • $LogRSyslogStatusMessages [on/off] - If set to on (the default), rsyslog emits message on startup and shutdown as well as when it is HUPed. This information might be needed by some log analyzers. If set to off, no such diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index 5ab73184..037e9f84 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -953,8 +953,6 @@ finalize_it: */ void dbgPrintCfSysLineHandlers(void) { - DEFiRet; - cslCmd_t *pCmd; cslCmdHdlr_t *pCmdHdlr; linkedListCookie_t llCookieCmd; @@ -976,7 +974,6 @@ void dbgPrintCfSysLineHandlers(void) } } dbgprintf("\n"); - ENDfunc } diff --git a/runtime/glbl.c b/runtime/glbl.c index 58558ed2..b396ed7c 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -64,6 +64,7 @@ static int option_DisallowWarning = 1; /* complain if message from disallowed se static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ static prop_t *propLocalHostName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup */ +static uchar *LocalHostNameOverride = NULL;/* user-overridden hostname - read-only after startup */ static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalDomain; /* our local domain name - read-only after startup */ static char **StripDomains = NULL;/* these domains may be stripped before writing logs - r/o after s.u., never touched by init */ @@ -153,17 +154,21 @@ GenerateLocalHostNameProperty(void) uchar *pszName; if(propLocalHostName != NULL) - prop.Destruct(&propLocalHostName); CHKiRet(prop.Construct(&propLocalHostName)); - if(LocalHostName == NULL) - pszName = (uchar*) "[localhost]"; - else { - if(GetPreserveFQDN() == 1) - pszName = LocalFQDNName; - else - pszName = LocalHostName; + if(LocalHostNameOverride == NULL) { + if(LocalHostName == NULL) + pszName = (uchar*) "[localhost]"; + else { + if(GetPreserveFQDN() == 1) + pszName = LocalFQDNName; + else + pszName = LocalHostName; + } + } else { /* local hostname is overriden via config */ + pszName = LocalHostNameOverride; } + DBGPRINTF("GenerateLocalHostName uses '%s'\n", pszName); CHKiRet(prop.SetString(propLocalHostName, pszName, ustrlen(pszName))); CHKiRet(prop.ConstructFinalize(propLocalHostName)); @@ -296,6 +301,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a free(pszDfltNetstrmDrvrCertFile); pszDfltNetstrmDrvrCertFile = NULL; } + if(LocalHostNameOverride != NULL) { + free(LocalHostNameOverride); + LocalHostNameOverride = NULL; + } if(pszWorkDir != NULL) { free(pszWorkDir); pszWorkDir = NULL; @@ -327,6 +336,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"localhostname", 0, eCmdHdlrGetWord, NULL, &LocalHostNameOverride, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL)); @@ -350,6 +360,7 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(pszWorkDir); if(LocalHostName != NULL) free(LocalHostName); + free(LocalHostNameOverride); if(LocalFQDNName != NULL) free(LocalFQDNName); objRelease(prop, CORE_COMPONENT); diff --git a/tests/testsuites/parse1.conf b/tests/testsuites/parse1.conf index 947a05a8..094cd762 100644 --- a/tests/testsuites/parse1.conf +++ b/tests/testsuites/parse1.conf @@ -2,6 +2,7 @@ $ModLoad ../plugins/omstdout/.libs/omstdout $IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ErrorMessagesToStderr off +$LocalHostName localhost # use a special format that we can easily parse in expect $template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" diff --git a/tools/syslogd.c b/tools/syslogd.c index 5f8d45a8..058d75d8 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2302,6 +2302,9 @@ init() legacyOptsHook(); + /* re-generate local host name property, as the config may have changed our FQDN settings */ + glbl.GenerateLocalHostNameProperty(); + /* we are now done with reading the configuration. This is the right time to * free some objects that were just needed for loading it. rgerhards 2005-10-19 */ @@ -3566,9 +3569,6 @@ int realMain(int argc, char **argv) if(!iConfigVerify) CHKiRet(doGlblProcessInit()); - /* re-generate local host name property, as the config may have changed our FQDN settings */ - glbl.GenerateLocalHostNameProperty(); - CHKiRet(mainThread()); /* do any de-init's that need to be done AFTER this comment */ -- cgit v1.2.3 From cc8237736d11b54a3d6089d836da7ccb6972a29c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Dec 2010 12:21:17 +0100 Subject: added $IMUDPSchedulingPolicy and $IMUDPSchedulingPriority config settings --- ChangeLog | 1 + plugins/imudp/imudp.c | 42 ++++++++++++++++++++++++++++++++++++++++++ runtime/glbl.c | 1 + 3 files changed, 44 insertions(+) diff --git a/ChangeLog b/ChangeLog index caa53b8c..740ed3c8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 4.7.4 [v4-???] (rgerhards), 2010-11-?? +- added $IMUDPSchedulingPolicy and $IMUDPSchedulingPriority config settings - added $LocalHostName config directive - bugfix: local hostname was pulled too-early, so that some config directives (namely FQDN settings) did not have any effect diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index d76f3544..fea789ec 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -72,6 +72,8 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * termination if we can not get it. -- rgerhards, 2007-12-27 */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ +static uchar *pszSchedPolicy = NULL; /**< scheduling policy (string) */ +static int iSchedPrio = -1; /**< scheduling priority (must not be negative) */ // TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -357,6 +359,7 @@ ENDrunInput /* initialize and return if will run or not */ BEGINwillRun + struct sched_param sparam; CODESTARTwillRun /* we need to create the inputName property (only once during our lifetime) */ CHKiRet(prop.Construct(&pInputName)); @@ -364,6 +367,41 @@ CODESTARTwillRun CHKiRet(prop.ConstructFinalize(pInputName)); net.PrintAllowedSenders(1); /* UDP */ + + if(pszSchedPolicy == NULL) { + if(iSchedPrio != -1) { + errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy not set, but " + "priority - ignoring settings"); + } + } else { + if(iSchedPrio == -1) { + errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy set, but no " + "priority - ignoring settings"); + } + sparam.sched_priority = iSchedPrio; + dbgprintf("imudp trying to set sched policy to '%s', prio %d\n", + pszSchedPolicy, iSchedPrio); + if(0) { /* trick to use conditional compilation */ +# ifdef SCHED_FIFO + } else if(!strcasecmp((char*)pszSchedPolicy, "fifo")) { + pthread_setschedparam(pthread_self(), SCHED_FIFO, &sparam); +# endif +# ifdef SCHED_RR + } else if(!strcasecmp((char*)pszSchedPolicy, "rr")) { + pthread_setschedparam(pthread_self(), SCHED_RR, &sparam); +# endif +# ifdef SCHED_OTHER + } else if(!strcasecmp((char*)pszSchedPolicy, "other")) { + pthread_setschedparam(pthread_self(), SCHED_OTHER, &sparam); +# endif + } else { + errmsg.LogError(errno, NO_ERRCODE, "imudp: invliad scheduling policy '%s' " + "ignoring settings", pszSchedPolicy); + } + free(pszSchedPolicy); + pszSchedPolicy = NULL; + } + /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) @@ -445,6 +483,10 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpolicy", 0, eCmdHdlrGetWord, + NULL, &pszSchedPolicy, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpriority", 0, eCmdHdlrInt, + NULL, &iSchedPrio, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, diff --git a/runtime/glbl.c b/runtime/glbl.c index b396ed7c..59d1fb0f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -154,6 +154,7 @@ GenerateLocalHostNameProperty(void) uchar *pszName; if(propLocalHostName != NULL) + prop.Destruct(&propLocalHostName); CHKiRet(prop.Construct(&propLocalHostName)); if(LocalHostNameOverride == NULL) { -- cgit v1.2.3 From 8d862730d297cbf88d9301b7a4cbee47a3f8fb9e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Dec 2010 14:59:44 +0100 Subject: added support for systems without epoll_create1() --- configure.ac | 2 +- plugins/imptcp/imptcp.c | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index b5c2d1c2..8e940e53 100644 --- a/configure.ac +++ b/configure.ac @@ -105,7 +105,7 @@ AC_TYPE_SIGNAL AC_FUNC_STAT AC_FUNC_STRERROR_R AC_FUNC_VPRINTF -AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline malloc_trim prctl fdatasync lseek64]) +AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline malloc_trim prctl epoll_create epoll_create1 fdatasync lseek64]) # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 93906ba0..2afb340f 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -30,6 +30,13 @@ * A copy of the GPL can be found in the file "COPYING" in this distribution. */ #include "config.h" +#if !defined(HAVE_EPOLL_CREATE) +# error imptcp requires OS support for epoll - can not build + /* imptcp gains speed by using modern Linux capabilities. As such, + * it can only be build on platforms supporting the epoll API. + */ +#endif + #include #include #include @@ -1040,7 +1047,18 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } - if((epollfd = epoll_create1(EPOLL_CLOEXEC)) < 0) { +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imptcp uses epoll_create1()\n"); + epollfd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("imptcp uses epoll_create()\n"); + /* reading the docs, the number of epoll events passed to + * epoll_create() seems not to be used at all in kernels. So + * we just provide "a" number, happens to be 10. + */ + epollfd = epoll_create(10); +# endif + if(epollfd < 0) { errmsg.LogError(0, RS_RET_EPOLL_CR_FAILED, "error: epoll_create() failed"); ABORT_FINALIZE(RS_RET_NO_RUN); } -- cgit v1.2.3 From dec726ed46a75750b30117ba772fc82e8660db1d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 10 Jan 2011 12:09:38 +0100 Subject: imudp: removing new scheduling priority setting code as it had too many implications. This will now be part of v5 only. --- ChangeLog | 1 - plugins/imudp/imudp.c | 42 ------------------------------------------ 2 files changed, 43 deletions(-) diff --git a/ChangeLog b/ChangeLog index 740ed3c8..caa53b8c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,5 @@ --------------------------------------------------------------------------- Version 4.7.4 [v4-???] (rgerhards), 2010-11-?? -- added $IMUDPSchedulingPolicy and $IMUDPSchedulingPriority config settings - added $LocalHostName config directive - bugfix: local hostname was pulled too-early, so that some config directives (namely FQDN settings) did not have any effect diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index fea789ec..d76f3544 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -72,8 +72,6 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * termination if we can not get it. -- rgerhards, 2007-12-27 */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -static uchar *pszSchedPolicy = NULL; /**< scheduling policy (string) */ -static int iSchedPrio = -1; /**< scheduling priority (must not be negative) */ // TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -359,7 +357,6 @@ ENDrunInput /* initialize and return if will run or not */ BEGINwillRun - struct sched_param sparam; CODESTARTwillRun /* we need to create the inputName property (only once during our lifetime) */ CHKiRet(prop.Construct(&pInputName)); @@ -367,41 +364,6 @@ CODESTARTwillRun CHKiRet(prop.ConstructFinalize(pInputName)); net.PrintAllowedSenders(1); /* UDP */ - - if(pszSchedPolicy == NULL) { - if(iSchedPrio != -1) { - errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy not set, but " - "priority - ignoring settings"); - } - } else { - if(iSchedPrio == -1) { - errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy set, but no " - "priority - ignoring settings"); - } - sparam.sched_priority = iSchedPrio; - dbgprintf("imudp trying to set sched policy to '%s', prio %d\n", - pszSchedPolicy, iSchedPrio); - if(0) { /* trick to use conditional compilation */ -# ifdef SCHED_FIFO - } else if(!strcasecmp((char*)pszSchedPolicy, "fifo")) { - pthread_setschedparam(pthread_self(), SCHED_FIFO, &sparam); -# endif -# ifdef SCHED_RR - } else if(!strcasecmp((char*)pszSchedPolicy, "rr")) { - pthread_setschedparam(pthread_self(), SCHED_RR, &sparam); -# endif -# ifdef SCHED_OTHER - } else if(!strcasecmp((char*)pszSchedPolicy, "other")) { - pthread_setschedparam(pthread_self(), SCHED_OTHER, &sparam); -# endif - } else { - errmsg.LogError(errno, NO_ERRCODE, "imudp: invliad scheduling policy '%s' " - "ignoring settings", pszSchedPolicy); - } - free(pszSchedPolicy); - pszSchedPolicy = NULL; - } - /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) @@ -483,10 +445,6 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpolicy", 0, eCmdHdlrGetWord, - NULL, &pszSchedPolicy, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpriority", 0, eCmdHdlrInt, - NULL, &iSchedPrio, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit v1.2.3 From 7742b2182fc017ca4c9fcfe3b26f4c8d68a9bd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dra=C5=BEen=20Ka=C4=8Dar?= Date: Mon, 10 Jan 2011 12:39:23 +0100 Subject: improved imudp real-time scheduling support & bugfix The original code had quite some issues, which are fixed by this commit. Also we do more error checking now. Signed-off-by: Rainer Gerhards --- configure.ac | 30 +++++++++++ plugins/imudp/Makefile.am | 2 +- plugins/imudp/imudp.c | 135 ++++++++++++++++++++++++++++++++++++++++++++++ runtime/rsyslog.c | 18 +++++++ runtime/rsyslog.h | 7 +++ runtime/stream.c | 8 ++- runtime/wtp.c | 6 +++ threads.c | 8 ++- 8 files changed, 211 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 22365513..02d7c342 100644 --- a/configure.ac +++ b/configure.ac @@ -269,6 +269,36 @@ if test "x$enable_pthreads" != "xno"; then ) fi +AC_CHECK_FUNCS( + [pthread_setschedparam], + [ + rsyslog_have_pthread_setschedparam=yes + ], + [ + rsyslog_have_pthread_setschedparam=no + ] +) +AC_CHECK_HEADERS( + [sched.h], + [ + rsyslog_have_sched_h=yes + ], + [ + rsyslog_have_sched_h=no + ] +) +if test "$rsyslog_have_pthread_setschedparam" = "yes" -a "$rsyslog_have_sched_h" = "yes"; then + save_LIBS=$LIBS + LIBS= + AC_SEARCH_LIBS(sched_get_priority_max, rt) + if test "x$ac_cv_search" != "xno"; then + AC_CHECK_FUNCS(sched_get_priority_max) + fi + IMUDP_LIBS=$LIBS + AC_SUBST(IMUDP_LIBS) + LIBS=$save_LIBS +fi + # klog AC_ARG_ENABLE(klog, diff --git a/plugins/imudp/Makefile.am b/plugins/imudp/Makefile.am index 517b1287..bc64b8c8 100644 --- a/plugins/imudp/Makefile.am +++ b/plugins/imudp/Makefile.am @@ -3,4 +3,4 @@ pkglib_LTLIBRARIES = imudp.la imudp_la_SOURCES = imudp.c imudp_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imudp_la_LDFLAGS = -module -avoid-version -imudp_la_LIBADD = +imudp_la_LIBADD = $(IMUDP_LIBS) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 99b69731..ad39ead0 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -35,6 +35,9 @@ #if HAVE_SYS_EPOLL_H # include #endif +#ifdef HAVE_SCHED_H +# include +#endif #include "rsyslog.h" #include "dirty.h" #include "net.h" @@ -78,12 +81,103 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * termination if we can not get it. -- rgerhards, 2007-12-27 */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ +static uchar *pszSchedPolicy = NULL; /* scheduling policy string */ +static int iSchedPolicy; /* scheduling policy as SCHED_xxx */ +static int iSchedPrio; /* scheduling priority */ +static int seen_iSchedPrio = 0; /* have we seen scheduling priority in the config file? */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ /* config settings */ +static rsRetVal check_scheduling_priority(int report_error) +{ + DEFiRet; + +#ifdef HAVE_SCHED_GET_PRIORITY_MAX + if (iSchedPrio < sched_get_priority_min(iSchedPolicy) || + iSchedPrio > sched_get_priority_max(iSchedPolicy)) { + if (report_error) + errmsg.LogError(errno, NO_ERRCODE, + "imudp: scheduling priority %d out of range (%d - %d)" + " for scheduling policy '%s' - ignoring settings", + iSchedPrio, + sched_get_priority_min(iSchedPolicy), + sched_get_priority_max(iSchedPolicy), + pszSchedPolicy); + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } +#endif + +finalize_it: + RETiRet; +} + +/* Set scheduling priority in the supplied variable (will be iSchedPrio) + * and record that we have seen the directive (in seen_iSchedPrio). + */ +static rsRetVal set_scheduling_priority(void *pVal, int value) +{ + DEFiRet; + + if (seen_iSchedPrio) { + errmsg.LogError(0, NO_ERRCODE, "directive already seen"); + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } + *(int *)pVal = value; + seen_iSchedPrio = 1; + if (pszSchedPolicy != NULL) + CHKiRet(check_scheduling_priority(1)); + +finalize_it: + RETiRet; +} + +/* Set scheduling policy in iSchedPolicy */ +static rsRetVal set_scheduling_policy(void *pVal, uchar *pNewVal) +{ + int have_sched_policy = 0; + DEFiRet; + + if (pszSchedPolicy != NULL) { + errmsg.LogError(0, NO_ERRCODE, "directive already seen"); + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } + *((uchar**)pVal) = pNewVal; /* pVal is pszSchedPolicy */ + if (0) { /* trick to use conditional compilation */ +#ifdef SCHED_FIFO + } else if (!strcasecmp((char*)pszSchedPolicy, "fifo")) { + iSchedPolicy = SCHED_FIFO; + have_sched_policy = 1; +#endif +#ifdef SCHED_RR + } else if (!strcasecmp((char*)pszSchedPolicy, "rr")) { + iSchedPolicy = SCHED_RR; + have_sched_policy = 1; +#endif +#ifdef SCHED_OTHER + } else if (!strcasecmp((char*)pszSchedPolicy, "other")) { + iSchedPolicy = SCHED_OTHER; + have_sched_policy = 1; +#endif + } else { + errmsg.LogError(errno, NO_ERRCODE, + "imudp: invalid scheduling policy '%s' " + "- ignoring setting", pszSchedPolicy); + } + if (have_sched_policy == 0) { + free(pszSchedPolicy); + pszSchedPolicy = NULL; + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } + if (seen_iSchedPrio) + CHKiRet(check_scheduling_priority(1)); + +finalize_it: + RETiRet; +} + /* This function is called when a new listener shall be added. It takes * the configured parameters, tries to bind the socket and, if that @@ -294,6 +388,41 @@ finalize_it: RETiRet; } +static void set_thread_schedparam(void) +{ + struct sched_param sparam; + + if (pszSchedPolicy != NULL && seen_iSchedPrio == 0) { + errmsg.LogError(0, NO_ERRCODE, + "imudp: scheduling policy set, but without priority - ignoring settings"); + } else if (pszSchedPolicy == NULL && seen_iSchedPrio != 0) { + errmsg.LogError(0, NO_ERRCODE, + "imudp: scheduling priority set, but without policy - ignoring settings"); + } else if (pszSchedPolicy != NULL && seen_iSchedPrio != 0 && + check_scheduling_priority(0) == 0) { +#ifndef HAVE_PTHREAD_SETSCHEDPARAM + errmsg.LogError(0, NO_ERRCODE, + "imudp: cannot set thread scheduling policy, " + "pthread_setschedparam() not available"); +#else + int err; + + memset(&sparam, 0, sizeof sparam); + sparam.sched_priority = iSchedPrio; + dbgprintf("imudp trying to set sched policy to '%s', prio %d\n", + pszSchedPolicy, iSchedPrio); + err = pthread_setschedparam(pthread_self(), iSchedPolicy, &sparam); + if (err != 0) { + errmsg.LogError(err, NO_ERRCODE, "imudp: pthread_setschedparam() failed"); + } +#endif + } + + if (pszSchedPolicy != NULL) { + free(pszSchedPolicy); + pszSchedPolicy = NULL; + } +} /* This function implements the main reception loop. Depending on the environment, * we either use the traditional (but slower) select() or the Linux-specific epoll() @@ -317,6 +446,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ + set_thread_schedparam(); bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); @@ -384,6 +514,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ + set_thread_schedparam(); bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); DBGPRINTF("imudp uses select()\n"); @@ -539,6 +670,10 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpolicy", 0, eCmdHdlrGetWord, + &set_scheduling_policy, &pszSchedPolicy, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpriority", 0, eCmdHdlrInt, + &set_scheduling_priority, &iSchedPrio, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index a9794840..2722fd6c 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -84,6 +84,12 @@ #include "strgen.h" #include "atomic.h" +#ifdef HAVE_PTHREAD_SETSCHEDPARAM +struct sched_param default_sched_param; +pthread_attr_t default_thread_attr; +int default_thr_sched_policy; +#endif + /* forward definitions */ static rsRetVal dfltErrLogger(int, uchar *errMsg); @@ -138,6 +144,18 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) if(iRefCount == 0) { /* init runtime only if not yet done */ +#ifdef HAVE_PTHREAD_SETSCHEDPARAM + CHKiRet(pthread_getschedparam(pthread_self(), + &default_thr_sched_policy, + &default_sched_param)); + CHKiRet(pthread_attr_init(&default_thread_attr)); + CHKiRet(pthread_attr_setschedpolicy(&default_thread_attr, + default_thr_sched_policy)); + CHKiRet(pthread_attr_setschedparam(&default_thread_attr, + &default_sched_param)); + CHKiRet(pthread_attr_setinheritsched(&default_thread_attr, + PTHREAD_EXPLICIT_SCHED)); +#endif if(ppErrObj != NULL) *ppErrObj = "obj"; CHKiRet(objClassInit(NULL)); /* *THIS* *MUST* always be the first class initilizer being called! */ CHKiRet(objGetObjInterface(pObjIF)); /* this provides the root pointer for all other queries */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 3e6ac4af..0742a2ed 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -25,6 +25,7 @@ */ #ifndef INCLUDED_RSYSLOG_H #define INCLUDED_RSYSLOG_H +#include #include "typedefs.h" /* ############################################################# * @@ -411,6 +412,12 @@ typedef enum rsObjectID rsObjID; #define RSFREEOBJ(x) {(x)->OID = OIDrsFreed; free(x);} #endif +#ifdef HAVE_PTHREAD_SETSCHEDPARAM +extern struct sched_param default_sched_param; +extern pthread_attr_t default_thread_attr; +extern int default_thr_sched_policy; +#endif + /* for the time being, we do our own portability handling here. It * looks like autotools either does not yet support checks for it, or diff --git a/runtime/stream.c b/runtime/stream.c index 260b59ef..658aba11 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -669,7 +669,13 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } pThis->pIOBuf = pThis->asyncBuf[0].pBuf; pThis->bStopWriter = 0; - if(pthread_create(&pThis->writerThreadID, NULL, asyncWriterThread, pThis) != 0) + if(pthread_create(&pThis->writerThreadID, +#ifdef HAVE_PTHREAD_SETSCHEDPARAM + &default_thread_attr, +#else + NULL, +#endif + asyncWriterThread, pThis) != 0) DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis); } else { /* we work synchronously, so we need to alloc a fixed pIOBuf */ diff --git a/runtime/wtp.c b/runtime/wtp.c index ece80911..e615fb19 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -90,6 +90,12 @@ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! pthread_mutex_init(&pThis->mutWtp, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); pthread_attr_init(&pThis->attrThrd); + /* Set thread scheduling policy to default */ +#ifdef HAVE_PTHREAD_SETSCHEDPARAM + pthread_attr_setschedpolicy(&pThis->attrThrd, default_thr_sched_policy); + pthread_attr_setschedparam(&pThis->attrThrd, &default_sched_param); + pthread_attr_setinheritsched(&pThis->attrThrd, PTHREAD_EXPLICIT_SCHED); +#endif pthread_attr_setdetachstate(&pThis->attrThrd, PTHREAD_CREATE_DETACHED); /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; diff --git a/threads.c b/threads.c index 4064fc7a..f1f275bd 100644 --- a/threads.c +++ b/threads.c @@ -220,7 +220,13 @@ rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdI pThis->pUsrThrdMain = thrdMain; pThis->pAfterRun = afterRun; pThis->bNeedsCancel = bNeedsCancel; - i = pthread_create(&pThis->thrdID, NULL, thrdStarter, pThis); + i = pthread_create(&pThis->thrdID, +#ifdef HAVE_PTHREAD_SETSCHEDPARAM + &default_thread_attr, +#else + NULL, +#endif + thrdStarter, pThis); CHKiRet(llAppend(&llThrds, NULL, pThis)); finalize_it: -- cgit v1.2.3 From 31125de72f3d787efb5b6ae8ba3117541b331bbc Mon Sep 17 00:00:00 2001 From: David Lang Date: Thu, 13 Jan 2011 09:08:33 +0100 Subject: added pmcisconames (uncompiled & untested) Signed-off-by: Rainer Gerhards --- plugins/pmcisconames/Makefile.am | 8 ++ plugins/pmcisconames/pmcisconames.c | 153 ++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 plugins/pmcisconames/Makefile.am create mode 100644 plugins/pmcisconames/pmcisconames.c diff --git a/plugins/pmcisconames/Makefile.am b/plugins/pmcisconames/Makefile.am new file mode 100644 index 00000000..16ed347d --- /dev/null +++ b/plugins/pmcisconames/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmcisconames.la + +pmcisconames_la_SOURCES = pmcisconames.c +pmcisconames_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmcisconames_la_LDFLAGS = -module -avoid-version +pmcisconames_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c new file mode 100644 index 00000000..28fe33d0 --- /dev/null +++ b/plugins/pmcisconames/pmcisconames.c @@ -0,0 +1,153 @@ +/* pmcisconames.c + * + * this detects logs sent by Cisco devices that mangle their syslog output when you tell them to log by name by adding ' :' between the name and the %XXX-X-XXXXXXX: tag + * + * instead of actually parsing the message, this modifies the message and then falls through to allow a later parser to handle the now modified message + * + * created 2010-12-13 by David Lang based on pmlastmsg + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.lastline") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINparse + uchar *p2parse; + int lenMsg; +#define OpeningText ": %" +CODESTARTparse + dbgprintf("Message will now be parsed by fix Cisco Names parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + + /* check if this message is of the type we handle in this (very limited) parser */ + /* first, we permit SP */ + while(lenMsg && *p2parse == ' ') { + --lenMsg; + ++p2parse; + } +dbgprintf("pmcisconames: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < 34) { + /* too short, can not be "our" message */ + /* minimum message, 16 character timestamp, 1 character name, ' : %ASA-1-000000: '*/ +dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* skip over timestamp */ + lenMsg -=16; + p2parse +=16; + /* now look for the next space to walk past the hostname */ + while(lenMsg && *p2parse != ' ') { + --lenMsg; + ++p2parse; + } + /* skip the space after the hostname */ + lenMsg -=1; + p2parse +=1; + /* if the syslog tag is : and the next thing starts with a % assume that this is a mangled cisco log and fix it */ + if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { + /* wrong opening text */ +dbgprintf("not a cisco name mangled log!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + /* bump the message portion up by two characters to overwrite the extra : */ + memmove(p2parse, p2parse + 2, lenMsg - 2); + /* now, claim to abort so that something else can parse the now modified message */ + DBGPRINTF("pmcisconames detected a mangled Cisco log message message\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ -- cgit v1.2.3 From 64a740a86162b19b3f57bcf221ce49aa6e57523a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 13 Jan 2011 09:16:50 +0100 Subject: integrated pmciscomsg into the build system --- Makefile.am | 4 ++++ configure.ac | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/Makefile.am b/Makefile.am index 680a819e..39b24f8a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -123,6 +123,10 @@ if ENABLE_OMSTDOUT SUBDIRS += plugins/omstdout endif +if ENABLE_PMCISCONAMES +SUBDIRS += plugins/pmcisconames +endif + if ENABLE_PMLASTMSG SUBDIRS += plugins/pmlastmsg endif diff --git a/configure.ac b/configure.ac index e66e6a4c..32c43aa1 100644 --- a/configure.ac +++ b/configure.ac @@ -937,6 +937,19 @@ AC_ARG_ENABLE(pmlastmsg, AM_CONDITIONAL(ENABLE_PMLASTMSG, test x$enable_pmlastmsg = xyes) +# settings for pmcisconames +AC_ARG_ENABLE(pmcisconames, + [AS_HELP_STRING([--enable-pmcisconames],[Compiles cisconames parser module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_pmcisconames="yes" ;; + no) enable_pmcisconames="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmcisconames) ;; + esac], + [enable_pmcisconames=no] +) +AM_CONDITIONAL(ENABLE_PMCISCONAMES, test x$enable_pmcisconames = xyes) + + # settings for pmrfc3164sd AC_ARG_ENABLE(pmrfc3164sd, [AS_HELP_STRING([--enable-pmrfc3164sd],[Compiles rfc3164sd parser module @<:@default=no@:>@])], @@ -1101,6 +1114,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omstdout/Makefile \ plugins/pmrfc3164sd/Makefile \ plugins/pmlastmsg/Makefile \ + plugins/pmcisconames/Makefile \ plugins/omruleset/Makefile \ plugins/omdbalerting/Makefile \ plugins/omuxsock/Makefile \ @@ -1161,6 +1175,7 @@ echo echo "---{ parser modules }---" echo " pmrfc3164sd module will be compiled: $enable_pmrfc3164sd" echo " pmlastmsg module will be compiled: $enable_pmlastmsg" +echo " pmcisconames module will be compiled: $enable_pmcisconames" echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" -- cgit v1.2.3 From 33e5e60734f654256ac33781728777798bc88d1c Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 25 Jan 2011 14:13:14 +0100 Subject: pmciscomsg: fixed some problems Signed-off-by: Rainer Gerhards --- plugins/pmcisconames/pmcisconames.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c index 28fe33d0..47d1f6f6 100644 --- a/plugins/pmcisconames/pmcisconames.c +++ b/plugins/pmcisconames/pmcisconames.c @@ -41,7 +41,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER -PARSER_NAME("rsyslog.lastline") +PARSER_NAME("rsyslog.cisconames") /* internal structures */ @@ -108,9 +108,14 @@ dbgprintf("not a cisco name mangled log!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } /* bump the message portion up by two characters to overwrite the extra : */ - memmove(p2parse, p2parse + 2, lenMsg - 2); + lenMsg -=2; + memmove(p2parse, p2parse + 2, lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=2; + pMsg->iLenMSG -=2; /* now, claim to abort so that something else can parse the now modified message */ - DBGPRINTF("pmcisconames detected a mangled Cisco log message message\n"); + DBGPRINTF("pmcisconames: new mesage: [%d]'%s'\n", lenMsg, p2parse); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); finalize_it: @@ -143,7 +148,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(parser, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); - dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION); + DBGPRINTF("cisconames parser init called, compiled with version %s\n", VERSION); bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ -- cgit v1.2.3 From 9813a12789d08a00fa96c11f113cd531c7f2ce7b Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 25 Jan 2011 14:24:50 +0100 Subject: enhanced imfile to support multi-line messages Signed-off-by: Rainer Gerhards --- ChangeLog | 1 + doc/imfile.html | 5 +++ plugins/imfile/imfile.c | 8 +++- runtime/stream.c | 111 ++++++++++++++++++++++++++++++++++++------------ runtime/stream.h | 5 ++- 5 files changed, 101 insertions(+), 29 deletions(-) diff --git a/ChangeLog b/ChangeLog index 18eb97b4..f4084f35 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 5.7.3 [V5-DEVEL] (rgerhards), 2010-12-?? +- added support for processing multi-line messages in imfile - added $IMUDPSchedulingPolicy and $IMUDPSchedulingPriority config settings - added $LocalHostName config directive - bugfix: fixed build problems on some platforms diff --git a/doc/imfile.html b/doc/imfile.html index f6b140a7..66c13e06 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -96,6 +96,11 @@ been processed. This setting can be used to guard against message duplication du to fatal errors (like power fail). Note that this setting affects imfile performance, especially when set to a low value. Frequently writing the state file is very time consuming. +
  • $InputFileReadMode [mode]
    +Available in 5.7.3+ +
    +Mode to be used when reading lines. 0 (the default) means that each line is forwarded +as its own log message. Caveats/Known Bugs:

    So far, only 100 files can be monitored. If more are needed, diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 8681ac8c..c205f60e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -71,6 +71,7 @@ typedef struct fileInfo_s { int nRecords; /**< How many records did we process before persisting the stream? */ int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ + int readMode; /* which mode to use in ReadMulteLine call? */ } fileInfo_t; @@ -85,6 +86,7 @@ static int iPollInterval = 10; /* number of seconds to sleep when there was no f static int iPersistStateInterval = 0; /* how often if state file to be persisted? (default 0->never) */ static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ +static int readMode = 0; /* mode to use for ReadMultiLine call */ static int iFilPtr = 0; /* number of files to be monitored; pointer to next free spot during config */ #define MAX_INPUT_FILES 100 @@ -212,7 +214,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) /* loop below will be exited when strmReadLine() returns EOF */ while(1) { - CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr)); + CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr, pThis->readMode)); *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ @@ -447,6 +449,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iPollInterval = 10; iFacility = 128; /* local0 */ iSeverity = 5; /* notice, as of rfc 3164 */ + readMode = 0; RETiRet; } @@ -489,6 +492,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->iFacility = iFacility; pThis->iPersistStateInterval = iPersistStateInterval; pThis->nRecords = 0; + pThis->readMode = readMode; iPersistStateInterval = 0; } else { errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one"); @@ -535,6 +539,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepollinterval", 0, eCmdHdlrInt, NULL, &iPollInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilereadmode", 0, eCmdHdlrInt, + NULL, &readMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); /* that command ads a new file! */ diff --git a/runtime/stream.c b/runtime/stream.c index 658aba11..16d41a2e 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -561,39 +561,98 @@ static rsRetVal strmUnreadChar(strm_t *pThis, uchar c) return RS_RET_OK; } - -/* read a line from a strm file. A line is terminated by LF. The LF is read, but it - * is not returned in the buffer (it is discared). The caller is responsible for - * destruction of the returned CStr object! -- rgerhards, 2008-01-07 - * rgerhards, 2008-03-27: I now use the ppCStr directly, without any interim - * string pointer. The reason is that this function my be called by inputs, which - * are pthread_killed() upon termination. So if we use their native pointer, they - * can cleanup (but only then). +/* read a 'paragraph' from a strm file. + * A paragraph may be terminated by a LF, by a LFLF, or by LF depending on the option set. + * The termination LF characters are read, but are + * not returned in the buffer (it is discared). The caller is responsible for + * destruction of the returned CStr object! -- dlang 2010-12-13 */ static rsRetVal -strmReadLine(strm_t *pThis, cstr_t **ppCStr) +strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) { - DEFiRet; - uchar c; - - ASSERT(pThis != NULL); - ASSERT(ppCStr != NULL); - - CHKiRet(cstrConstruct(ppCStr)); - - /* now read the line */ - CHKiRet(strmReadChar(pThis, &c)); - while(c != '\n') { - CHKiRet(cstrAppendChar(*ppCStr, c)); - CHKiRet(strmReadChar(pThis, &c)); + /* mode = 0 single line mode (equivalent to ReadLine) + * mode = 1 LFLF mode (paragraph, blank line between entries) + * mode = 2 LF mode, a log line starts at the beginning of a line, but following lines that are indented are part of the same log entry + * This modal interface is not nearly as flexible as being able to define a regex for when a new record starts, but it's also not nearly as hard (or as slow) to implement + */ + DEFiRet; + uchar c; + uchar finished; + + ASSERT(pThis != NULL); + ASSERT(ppCStr != NULL); + + CHKiRet(cstrConstruct(ppCStr)); + + /* now read the line */ + CHKiRet(strmReadChar(pThis, &c)); + if (mode == 0){ + while(c != '\n') { + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } + CHKiRet(cstrFinalize(*ppCStr)); + } + if (mode == 1){ + finished=0; + while(finished == 0){ + if(c != '\n') { + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { + if ((((*ppCStr)->iStrLen) > 0) ){ + if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] == '\n'){ + rsCStrTruncate(*ppCStr,1); /* remove the prior newline */ + finished=1; + } else { + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } + } else { + finished=1; /* this is a blank line, a \n with nothing since the last complete record */ + } + } + } + CHKiRet(cstrFinalize(*ppCStr)); + } + if (mode == 2){ +/* indented follow-up lines */ + finished=0; + while(finished == 0){ + if ((*ppCStr)->iStrLen == 0){ + if(c != '\n') { +/* nothing in the buffer, and it's not a newline, add it to the buffer */ + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { + finished=1; /* this is a blank line, a \n with nothing since the last complete record */ + } + } else { + if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] != '\n'){ +/* not the first character after a newline, add it to the buffer */ + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { + if ((c == ' ') || (c == '\t')){ + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { +/* clean things up by putting the character we just read back into the input buffer and removing the LF character that is currently at the end of the output string */ + CHKiRet(strmUnreadChar(pThis, c)); + rsCStrTruncate(*ppCStr,1); + finished=1; + } + } + } + } + CHKiRet(cstrFinalize(*ppCStr)); } - CHKiRet(cstrFinalize(*ppCStr)); finalize_it: - if(iRet != RS_RET_OK && *ppCStr != NULL) - cstrDestruct(ppCStr); + if(iRet != RS_RET_OK && *ppCStr != NULL) + cstrDestruct(ppCStr); - RETiRet; + RETiRet; } diff --git a/runtime/stream.h b/runtime/stream.h index 37e9d570..60c68cb2 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -156,7 +156,6 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); rsRetVal (*ReadChar)(strm_t *pThis, uchar *pC); rsRetVal (*UnreadChar)(strm_t *pThis, uchar c); - rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr); rsRetVal (*SeekCurrOffs)(strm_t *pThis); rsRetVal (*Write)(strm_t *pThis, uchar *pBuf, size_t lenBuf); rsRetVal (*WriteChar)(strm_t *pThis, uchar c); @@ -183,8 +182,10 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, iSizeLimit, off_t); INTERFACEpropSetMeth(strm, iFlushInterval, int); INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); + /* v6 added */ + rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr, int mode); ENDinterface(strm) -#define strmCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* prototypes */ -- cgit v1.2.3 From 44e3fb96c5dd94a0669beff70cfa9d9097c35de2 Mon Sep 17 00:00:00 2001 From: Ariel P Date: Mon, 31 Jan 2011 17:22:39 +0100 Subject: added $OMMySQLConfigFile/$OMMySQLConfigSection config directives Signed-off-by: Rainer Gerhards --- ChangeLog | 2 ++ plugins/ommysql/ommysql.c | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/ChangeLog b/ChangeLog index 0c66442f..5052a565 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,8 @@ Version 5.7.3 [V5-DEVEL] (rgerhards), 2010-12-?? - bugfix: local hostname was pulled too-early, so that some config directives (namely FQDN settings) did not have any effect - bugfix: imfile did duplicate messages under some circumstances +- added $OMMySQLConfigFile config directive +- added $OMMySQLConfigSection config directive --------------------------------------------------------------------------- Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-11-26 - bugfix(important): problem in TLS handling could cause rsyslog to loop diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index d6870a7b..aff76d0a 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -60,9 +60,13 @@ typedef struct _instanceData { char f_dbuid[_DB_MAXUNAMELEN+1]; /* DB user */ char f_dbpwd[_DB_MAXPWDLEN+1]; /* DB user's password */ unsigned uLastMySQLErrno; /* last errno returned by MySQL or 0 if all is well */ + uchar * f_configfile; /* MySQL Client Configuration File */ + uchar * f_configsection; /* MySQL Client Configuration Section */ } instanceData; /* config variables */ +static uchar * pszMySQLConfigFile = NULL; /* MySQL Client Configuration File */ +static uchar * pszMySQLConfigSection = NULL; /* MySQL Client Configuration Section */ static int iSrvPort = 0; /* database server port */ @@ -91,6 +95,14 @@ static void closeMySQL(instanceData *pData) mysql_close(pData->f_hmysql); pData->f_hmysql = NULL; } + if(pData->f_configfile!=NULL){ + free(pData->f_configfile); + pData->f_configfile=NULL; + } + if(pData->f_configsection!=NULL){ + free(pData->f_configsection); + pData->f_configsection=NULL; + } } BEGINfreeInstance @@ -152,6 +164,25 @@ static rsRetVal initMySQL(instanceData *pData, int bSilent) errmsg.LogError(0, RS_RET_SUSPENDED, "can not initialize MySQL handle"); iRet = RS_RET_SUSPENDED; } else { /* we could get the handle, now on with work... */ + mysql_options(pData->f_hmysql,MYSQL_READ_DEFAULT_GROUP,((pData->f_configsection!=NULL)?(char*)pData->f_configsection:"client")); + if(pData->f_configfile!=NULL){ + FILE * fp; + fp=fopen((char*)pData->f_configfile,"r"); + int err=errno; + if(fp==NULL){ + char msg[512]; + snprintf(msg,sizeof(msg)/sizeof(char),"Could not open '%s' for reading",pData->f_configfile); + if(bSilent) { + char errStr[512]; + rs_strerror_r(err, errStr, sizeof(errStr)); + dbgprintf("mysql configuration error(%d): %s - %s\n",err,msg,errStr); + } else + errmsg.LogError(err,NO_ERRCODE,"mysql configuration error: %s\n",msg); + } else { + fclose(fp); + mysql_options(pData->f_hmysql,MYSQL_READ_DEFAULT_FILE,pData->f_configfile); + } + } /* Connect to database */ if(mysql_real_connect(pData->f_hmysql, pData->f_dbsrv, pData->f_dbuid, pData->f_dbpwd, pData->f_dbname, pData->f_dbsrvPort, NULL, 0) == NULL) { @@ -278,6 +309,8 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } else { pData->f_dbsrvPort = (unsigned) iSrvPort; /* set configured port */ + pData->f_configfile = pszMySQLConfigFile; + pData->f_configsection = pszMySQLConfigSection; pData->f_hmysql = NULL; /* initialize, but connect only on first message (important for queued mode!) */ } @@ -302,6 +335,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a { DEFiRet; iSrvPort = 0; /* zero is the default port */ + free(pszMySQLConfigFile); + pszMySQLConfigFile = NULL; + free(pszMySQLConfigSection); + pszMySQLConfigSection = NULL; RETiRet; } @@ -313,6 +350,8 @@ CODEmodInit_QueryRegCFSLineHdlr /* register our config handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionommysqlserverport", 0, eCmdHdlrInt, NULL, &iSrvPort, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"ommysqlconfigfile",0,eCmdHdlrGetWord,NULL,&pszMySQLConfigFile,STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"ommysqlconfigsection",0,eCmdHdlrGetWord,NULL,&pszMySQLConfigSection,STD_LOADABLE_MODULE_ID)); ENDmodInit /* vi:set ai: -- cgit v1.2.3 From 58cb4d57e81da5aa74ed2927f99b576cec83d71b Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 1 Feb 2011 11:36:27 +0100 Subject: added pmaixforwardedfrom message parser Signed-off-by: Rainer Gerhards --- Makefile.am | 4 + configure.ac | 15 +++ plugins/pmaixforwardedfrom/Makefile.am | 8 ++ plugins/pmaixforwardedfrom/pmaixforwardedfrom.c | 167 ++++++++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 plugins/pmaixforwardedfrom/Makefile.am create mode 100644 plugins/pmaixforwardedfrom/pmaixforwardedfrom.c diff --git a/Makefile.am b/Makefile.am index 39b24f8a..f6a87e6a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -127,6 +127,10 @@ if ENABLE_PMCISCONAMES SUBDIRS += plugins/pmcisconames endif +if ENABLE_PMAIXFORWARDEDFROM +SUBDIRS += plugins/pmaixforwardedfrom +endif + if ENABLE_PMLASTMSG SUBDIRS += plugins/pmlastmsg endif diff --git a/configure.ac b/configure.ac index 32c43aa1..da939c17 100644 --- a/configure.ac +++ b/configure.ac @@ -950,6 +950,19 @@ AC_ARG_ENABLE(pmcisconames, AM_CONDITIONAL(ENABLE_PMCISCONAMES, test x$enable_pmcisconames = xyes) +# settings for pmaixforwardedfrom +AC_ARG_ENABLE(pmaixforwardedfrom, + [AS_HELP_STRING([--enable-pmaixforwardedfrom],[Compiles aixforwardedfrom parser module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_pmaixforwardedfrom="yes" ;; + no) enable_pmaixforwardedfrom="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmaixforwardedfrom) ;; + esac], + [enable_pmaixforwardedfrom=no] +) +AM_CONDITIONAL(ENABLE_PMAIXFORWARDEDFROM, test x$enable_pmaixforwardedfrom = xyes) + + # settings for pmrfc3164sd AC_ARG_ENABLE(pmrfc3164sd, [AS_HELP_STRING([--enable-pmrfc3164sd],[Compiles rfc3164sd parser module @<:@default=no@:>@])], @@ -1115,6 +1128,7 @@ AC_CONFIG_FILES([Makefile \ plugins/pmrfc3164sd/Makefile \ plugins/pmlastmsg/Makefile \ plugins/pmcisconames/Makefile \ + plugins/pmaixforwardedfrom/Makefile \ plugins/omruleset/Makefile \ plugins/omdbalerting/Makefile \ plugins/omuxsock/Makefile \ @@ -1176,6 +1190,7 @@ echo "---{ parser modules }---" echo " pmrfc3164sd module will be compiled: $enable_pmrfc3164sd" echo " pmlastmsg module will be compiled: $enable_pmlastmsg" echo " pmcisconames module will be compiled: $enable_pmcisconames" +echo " pmaixforwardedfrom module will be compiled: $enable_pmaixforwardedfrom" echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" diff --git a/plugins/pmaixforwardedfrom/Makefile.am b/plugins/pmaixforwardedfrom/Makefile.am new file mode 100644 index 00000000..af359d31 --- /dev/null +++ b/plugins/pmaixforwardedfrom/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmaixforwardedfrom.la + +pmaixforwardedfrom_la_SOURCES = pmaixforwardedfrom.c +pmaixforwardedfrom_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmaixforwardedfrom_la_LDFLAGS = -module -avoid-version +pmaixforwardedfrom_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c new file mode 100644 index 00000000..11634199 --- /dev/null +++ b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c @@ -0,0 +1,167 @@ +/* pmaixforwardedfrom.c + * + * this detects logs sent by Cisco devices that mangle their syslog output when you tell them to log by name by adding ' :' between the name and the %XXX-X-XXXXXXX: tag + * + * instead of actually parsing the message, this modifies the message and then falls through to allow a later parser to handle the now modified message + * + * created 2010-12-13 by David Lang based on pmlastmsg + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.aixforwardedfrom") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINparse + uchar *p2parse; + uchar *opening; + int lenMsg; +#define OpeningText "Message forwarded from " +CODESTARTparse + dbgprintf("Message will now be parsed by fix AIX Forwarded From parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + + /* check if this message is of the type we handle in this (very limited) parser */ + /* first, we permit SP */ + while(lenMsg && *p2parse == ' ') { + --lenMsg; + ++p2parse; + } +dbgprintf("pmaixforwardedfrom: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < 42) { + /* too short, can not be "our" message */ + /* minimum message, 16 character timestamp, 'Message forwarded from ", 1 character name, ': '*/ +dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* skip over timestamp */ + lenMsg -=16; + p2parse +=16; + /* if there is the string "Message forwarded from " were the hostname should be */ + if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { + /* wrong opening text */ +dbgprintf("not a AIX message forwarded from mangled log!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + /* bump the message portion up by 23 characters to overwrite the "Message forwarded from " with the hostname */ + lenMsg -=23; + memmove(p2parse, p2parse + 23, lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=23; + pMsg->iLenMSG -=23; + /* now look for the : after the hostname to walk past the hostname, also watch for a space in case this isn't really an AIX log, but has a similar preamble */ + while(lenMsg && *p2parse != ' ' && *p2parse != ':') { + --lenMsg; + ++p2parse; + } + if (lenMsg && *p2parse != ':') { +dbgprintf("not a AIX message forwarded from mangled log but similar enough that the preamble has been removed\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + /* bump the message portion up by one character to overwrite the extra : */ + lenMsg -=1; + memmove(p2parse, p2parse + 1, lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=1; + pMsg->iLenMSG -=1; + /* now, claim to abort so that something else can parse the now modified message */ + DBGPRINTF("pmaixforwardedfrom: new mesage: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + DBGPRINTF("aixforwardedfrom parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ -- cgit v1.2.3 From a1f4330a7b1ab426d7abeeacb4ff4e5994c429e6 Mon Sep 17 00:00:00 2001 From: Ariel P Date: Tue, 1 Feb 2011 11:50:35 +0100 Subject: added doc for new ommysql directives --- doc/ommysql.html | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/ommysql.html b/doc/ommysql.html index 9b35b402..daef9cab 100644 --- a/doc/ommysql.html +++ b/doc/ommysql.html @@ -24,6 +24,18 @@ directive configuration system. a non-standard port for the MySQL server. The default is 0, which means the system default port is used. There is no need to specify this directive unless you know the server is running on a non-standard listen port. +

  • $OmMySQLConfigFile <file name>
    Permits the selection +of an optional MySQL Client Library configuration file (my.cnf) for extended +configuration functionality. The use of this configuration directive is necessary +only if you have a non-standard environment or if fine-grained control over the +database connection is desired.
    • +
  • $OmMySQLConfigSection <string>
    Permits the selection of the +section within the configuration file specified by the $OmMySQLConfigFile directive. +
    This will likely only be used where the database administrator provides a single +configuration file with multiple profiles. +
    This configuration directive is ignored unless $OmMySQLConfigFile is also used +in the rsyslog configration file. +
    If omitted, the MySQL Client Library default of "client" will be used.
  • Action parameters:
    :ommysql:database-server,database-name,database-userid,database-password
    All parameters should be filled in for a successful connect. -- cgit v1.2.3 From 024fdc16f7e7e19040153ff28475105f174ae597 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 4 Feb 2011 09:24:31 +0100 Subject: removed invalid link from doc --- doc/manual.html | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual.html b/doc/manual.html index 7fdf3f14..f63e22a3 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -41,7 +41,6 @@ if you do not read the doc, but doing so will definitely improve your experience
  • configuration file syntax (rsyslog.conf)
  • a regular expression checker/generator tool for rsyslog
  • property replacer, an important core component
    • -
  • a commented sample rsyslog.conf
  • rsyslog bug list
  • understanding rsyslog message parsers
  • backgrounder on generic syslog application design
    • -- cgit v1.2.3 From 0a36190a5ddb3c87d22453547d9a4840d0f27cf7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 4 Feb 2011 16:42:17 +0100 Subject: preparing for 5.7.3 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5052a565..660887a9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.3 [V5-DEVEL] (rgerhards), 2010-12-?? +Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 - added support for processing multi-line messages in imfile - added $IMUDPSchedulingPolicy and $IMUDPSchedulingPriority config settings - added $LocalHostName config directive diff --git a/configure.ac b/configure.ac index da939c17..d5d37b2b 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.3],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index f63e22a3..e8a1ccfa 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.2 (beta branch) of rsyslog. +

    This documentation is for version 5.7.3 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From fa89eef48e25608585454915ff7af9fae524042e Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 8 Feb 2011 11:46:05 +0100 Subject: added pmsnare parser module Signed-off-by: Rainer Gerhards --- ChangeLog | 1 + Makefile.am | 4 + configure.ac | 17 +++- plugins/pmsnare/Makefile.am | 8 ++ plugins/pmsnare/pmsnare.c | 233 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 plugins/pmsnare/Makefile.am create mode 100644 plugins/pmsnare/pmsnare.c diff --git a/ChangeLog b/ChangeLog index 660887a9..d62eb3f7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +- added pmsnare parser module (written by David Lang) --------------------------------------------------------------------------- Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 - added support for processing multi-line messages in imfile diff --git a/Makefile.am b/Makefile.am index f6a87e6a..9493e373 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,6 +131,10 @@ if ENABLE_PMAIXFORWARDEDFROM SUBDIRS += plugins/pmaixforwardedfrom endif +if ENABLE_PMSNARE +SUBDIRS += plugins/pmsnare +endif + if ENABLE_PMLASTMSG SUBDIRS += plugins/pmlastmsg endif diff --git a/configure.ac b/configure.ac index d5d37b2b..ffabb93c 100644 --- a/configure.ac +++ b/configure.ac @@ -963,6 +963,19 @@ AC_ARG_ENABLE(pmaixforwardedfrom, AM_CONDITIONAL(ENABLE_PMAIXFORWARDEDFROM, test x$enable_pmaixforwardedfrom = xyes) +# settings for pmsnare +AC_ARG_ENABLE(pmsnare, + [AS_HELP_STRING([--enable-pmsnare],[Compiles snare parser module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_pmsnare="yes" ;; + no) enable_pmsnare="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmsnare) ;; + esac], + [enable_pmsnare=no] +) +AM_CONDITIONAL(ENABLE_PMSNARE, test x$enable_pmsnare = xyes) + + # settings for pmrfc3164sd AC_ARG_ENABLE(pmrfc3164sd, [AS_HELP_STRING([--enable-pmrfc3164sd],[Compiles rfc3164sd parser module @<:@default=no@:>@])], @@ -1128,6 +1141,7 @@ AC_CONFIG_FILES([Makefile \ plugins/pmrfc3164sd/Makefile \ plugins/pmlastmsg/Makefile \ plugins/pmcisconames/Makefile \ + plugins/pmsnare/Makefile \ plugins/pmaixforwardedfrom/Makefile \ plugins/omruleset/Makefile \ plugins/omdbalerting/Makefile \ @@ -1190,7 +1204,8 @@ echo "---{ parser modules }---" echo " pmrfc3164sd module will be compiled: $enable_pmrfc3164sd" echo " pmlastmsg module will be compiled: $enable_pmlastmsg" echo " pmcisconames module will be compiled: $enable_pmcisconames" -echo " pmaixforwardedfrom module will be compiled: $enable_pmaixforwardedfrom" +echo " pmaixforwardedfrom module w.be compiled: $enable_pmaixforwardedfrom" +echo " pmsnare module will be compiled: $enable_pmsnare" echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" diff --git a/plugins/pmsnare/Makefile.am b/plugins/pmsnare/Makefile.am new file mode 100644 index 00000000..5b2696ac --- /dev/null +++ b/plugins/pmsnare/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmsnare.la + +pmsnare_la_SOURCES = pmsnare.c +pmsnare_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmsnare_la_LDFLAGS = -module -avoid-version +pmsnare_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmsnare/pmsnare.c b/plugins/pmsnare/pmsnare.c new file mode 100644 index 00000000..ce2138b2 --- /dev/null +++ b/plugins/pmsnare/pmsnare.c @@ -0,0 +1,233 @@ +/* pmsnare.c + * + * this detects logs sent by Snare and cleans them up so that they can be processed by the normal parser + * + * there are two variations of this, if the client is set to 'syslog' mode it sends + * + * timestamphostnametagotherstuff + * + * if the client is not set to syslog it sends + * + * hostnametagotherstuff + * + * ToDo, take advantage of items in the message itself to set more friendly information + * where the normal parser will find it by re-writing more of the message + * + * Intereting information includes: + * + * in the case of windows snare messages: + * the system hostname is field 12 + * the severity is field 3 (criticality ranging form 0 to 4) + * the source of the log is field 4 and may be able to be mapped to facility + * + * + * created 2010-12-13 by David Lang based on pmlastmsg + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.snare") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINparse + uchar *p2parse; + int lenMsg; + int snaremessage; + int tablength; + +CODESTARTparse + #define TabRepresentation "#011" + tablength=sizeof(TabRepresentation); + dbgprintf("Message will now be parsed by fix Snare parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + + /* check if this message is of the type we handle in this (very limited) parser + + find out if we have a space separated or tab separated for the first item + if tab separated see if the second word is one of our expected tags + if so replace the tabs with spaces so that hostname and syslog tag are going to be parsed properly + optionally replace the hostname at the beginning of the message with one from later in the message + else, wrong message, abort + else, assume that we have a valid timestamp, move over to the syslog tag + if that is tab separated from the rest of the message and one of our expected tags + if so, replace the tab with a space so that it will be parsed properly + optionally replace the hostname at the beginning of the message withone from later in the message + + */ + snaremessage=0; + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + dbgprintf("pmsnare: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < 30) { + /* too short, can not be "our" message */ + dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + while(lenMsg && *p2parse != ' ' && *p2parse != '\t' && *p2parse != '#') { + --lenMsg; + ++p2parse; + } + dbgprintf("pmsnare: separator [%d]'%s' msg after the first separator: [%d]'%s'\n", tablength,TabRepresentation,lenMsg, p2parse); + if ((lenMsg > tablength) && (*p2parse == '\t' || strncasecmp((char*) p2parse, TabRepresentation , tablength-1) == 0)) { + //if ((lenMsg > tablength) && (*p2parse == '\t' || *p2parse == '#')) { + dbgprintf("pmsnare: tab separated message\n"); + if(strncasecmp((char*) (p2parse + tablength - 1), "MSWinEventLog", 13) == 0) { + snaremessage=13; /* 0 means not a snare message, a number is how long the tag is */ + } + if(strncasecmp((char*) (p2parse + tablength - 1), "LinuxKAudit", 11) == 0) { + snaremessage=11; /* 0 means not a snare message, a number is how long the tag is */ + } + if(snaremessage) { + /* replace the tab with a space and if needed move the message portion up by the length of TabRepresentation -2 characters to overwrite the extra : */ + *p2parse = ' '; + lenMsg -=(tablength-2); + p2parse++; + memmove(p2parse, p2parse + (tablength-2), lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=(tablength-2); + pMsg->iLenMSG -=(tablength-2); + p2parse += snaremessage; + *p2parse = ' '; + p2parse++; + lenMsg -=(tablength-2); + memmove(p2parse, p2parse + (tablength-2), lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=(tablength-2); + pMsg->iLenMSG -=(tablength-2); + dbgprintf("found a Snare message with snare not set to send syslog messages\n"); + } + } else { + /* go back to the beginning of the message */ + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + /* skip over timestamp and space*/ + lenMsg -=17; + p2parse +=17; + /* skip over what should be the hostname */ + while(lenMsg && *p2parse != ' ') { + --lenMsg; + ++p2parse; + } + if (lenMsg){ + --lenMsg; + ++p2parse; + } + dbgprintf("pmsnare: separator [%d]'%s' msg after the timestamp and hostname: [%d]'%s'\n", tablength,TabRepresentation,lenMsg, p2parse); + if(lenMsg > 13 && strncasecmp((char*) p2parse, "MSWinEventLog", 13) == 0) { + snaremessage=13; /* 0 means not a snare message, a number is how long the tag is */ + } + if(lenMsg > 11 && strncasecmp((char*) p2parse, "LinuxKAudit", 11) == 0) { + snaremessage=11; /* 0 means not a snare message, a number is how long the tag is */ + } + if(snaremessage) { + p2parse += snaremessage; + *p2parse = ' '; + p2parse++; + lenMsg -=(tablength-2); + memmove(p2parse, p2parse + (tablength-2), lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=(tablength-2); + pMsg->iLenMSG -=(tablength-2); + dbgprintf("found a Snare message with snare set to send syslog messages\n"); + } + + } + DBGPRINTF("pmsnare: new mesage: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + DBGPRINTF("snare parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ -- cgit v1.2.3 From ff686d8021eb739802009db51998fd9731c0dc8a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 8 Feb 2011 12:16:25 +0100 Subject: prepare for new version --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index d62eb3f7..2960749e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +--------------------------------------------------------------------------- +Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? - added pmsnare parser module (written by David Lang) --------------------------------------------------------------------------- Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 -- cgit v1.2.3 From 7de7360ede231f0befdbdeecedcbd310e74a0f26 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 8 Feb 2011 18:01:17 +0100 Subject: fixed invalid help strings in configure.ac Thanks to Michael Biebl for mentioning this problem! --- configure.ac | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index ffabb93c..8eb840b0 100644 --- a/configure.ac +++ b/configure.ac @@ -801,7 +801,7 @@ AC_SUBST(LIBLOGGING_LIBS) # enable/disable the testbench (e.g. because some important parts # are missing) AC_ARG_ENABLE(testbench, - [AS_HELP_STRING([--enable-testbench],[file input module enabled @<:@default=yes@:>@])], + [AS_HELP_STRING([--enable-testbench],[testbench enabled @<:@default=yes@:>@])], [case "${enableval}" in yes) enable_testbench="yes" ;; no) enable_testbench="no" ;; @@ -832,7 +832,7 @@ AM_CONDITIONAL(ENABLE_IMFILE, test x$enable_imfile = xyes) # settings for the door input module (under solaris, thus default off) AC_ARG_ENABLE(imsolaris, - [AS_HELP_STRING([--enable-imsolaris],[door input module enabled @<:@default=no@:>@])], + [AS_HELP_STRING([--enable-imsolaris],[solaris input module enabled @<:@default=no@:>@])], [case "${enableval}" in yes) enable_imsolaris="yes" ;; no) enable_imsolaris="no" ;; @@ -1039,7 +1039,7 @@ AC_SUBST(RELP_LIBS) # settings for omuxsock AC_ARG_ENABLE(omuxsock, - [AS_HELP_STRING([--enable-omuxsock],[Compiles stdout module @<:@default=no@:>@])], + [AS_HELP_STRING([--enable-omuxsock],[Compiles omuxsock module @<:@default=no@:>@])], [case "${enableval}" in yes) enable_omuxsock="yes" ;; no) enable_omuxsock="no" ;; @@ -1054,7 +1054,7 @@ AM_CONDITIONAL(ENABLE_OMUXSOCK, test x$enable_omuxsock = xyes) # part of rsyslog, into the build process. It is named cust1, so that # additional such modules can easily be added. AC_ARG_ENABLE(cust1, - [AS_HELP_STRING([--enable-cust1],[Compiles stdout module @<:@default=no@:>@])], + [AS_HELP_STRING([--enable-cust1],[Compiles cust1 module @<:@default=no@:>@])], [case "${enableval}" in yes) enable_cust1="yes" ;; no) enable_cust1="no" ;; -- cgit v1.2.3 From fce5fe5127c4b2e4ba17a64b8fb9d57b573baec0 Mon Sep 17 00:00:00 2001 From: David Lang Date: Thu, 10 Feb 2011 07:36:34 +0100 Subject: pmcisconames: bugfix for short timestamps Signed-off-by: Rainer Gerhards --- plugins/pmcisconames/pmcisconames.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c index 47d1f6f6..4171e688 100644 --- a/plugins/pmcisconames/pmcisconames.c +++ b/plugins/pmcisconames/pmcisconames.c @@ -89,10 +89,29 @@ dbgprintf("pmcisconames: msg to look at: [%d]'%s'\n", lenMsg, p2parse); dbgprintf("msg too short!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } - - /* skip over timestamp */ - lenMsg -=16; - p2parse +=16; + /* check if the timestamp is a 16 character or 21 character timestamp + 'Mmm DD HH:MM:SS ' spaces at 3,6,15 : at 9,12 + 'Mmm DD YYYY HH:MM:SS ' spaces at 3,6,11,20 : at 14,17 + check for the : first as that will differentiate the two conditions the fastest + this allows the compiler to short circuit the rst of the tests if it is the wrong timestamp + but still check the rest to see if it looks correct + */ + if ( *(p2parse + 9) == ':' && *(p2parse + 12) == ':' && *(p2parse + 3) == ' ' && *(p2parse + 6) == ' ' && *(p2parse + 15) == ' ') { + /* skip over timestamp */ + dbgprintf("short timestamp found\n"); + lenMsg -=16; + p2parse +=16; + } else { + if ( *(p2parse + 14) == ':' && *(p2parse + 17) == ':' && *(p2parse + 3) == ' ' && *(p2parse + 6) == ' ' && *(p2parse + 11) == ' ' && *(p2parse + 20) == ' ') { + /* skip over timestamp */ + dbgprintf("long timestamp found\n"); + lenMsg -=21; + p2parse +=21; + } else { + dbgprintf("timestamp is not one of the valid formats\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + } /* now look for the next space to walk past the hostname */ while(lenMsg && *p2parse != ' ') { --lenMsg; -- cgit v1.2.3 From df6bad70d4df6b9142f988bd4ed859555756fd15 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Feb 2011 17:46:04 +0100 Subject: cosmetic: making comments in stream.c a bit more readable --- runtime/stream.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/runtime/stream.c b/runtime/stream.c index 5f4249a8..24dbcc09 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -617,12 +617,12 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) CHKiRet(cstrFinalize(*ppCStr)); } if (mode == 2){ -/* indented follow-up lines */ + /* indented follow-up lines */ finished=0; while(finished == 0){ if ((*ppCStr)->iStrLen == 0){ if(c != '\n') { -/* nothing in the buffer, and it's not a newline, add it to the buffer */ + /* nothing in the buffer, and it's not a newline, add it to the buffer */ CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } else { @@ -630,7 +630,7 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) } } else { if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] != '\n'){ -/* not the first character after a newline, add it to the buffer */ + /* not the first character after a newline, add it to the buffer */ CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } else { @@ -638,7 +638,9 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } else { -/* clean things up by putting the character we just read back into the input buffer and removing the LF character that is currently at the end of the output string */ + /* clean things up by putting the character we just read back into + * the input buffer and removing the LF character that is currently at the + * end of the output string */ CHKiRet(strmUnreadChar(pThis, c)); rsCStrTruncate(*ppCStr,1); finished=1; -- cgit v1.2.3 From 1f66eb6efaf2c4bea18242645c85b51fe5236fb9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Feb 2011 17:50:41 +0100 Subject: bugfix: abort if imfile reads file line of more than 64KiB Thanks to Peter Eisentraut for reporting and analysing this problem. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=221 --- ChangeLog | 3 +++ runtime/stringbuf.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2960749e..15367918 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,9 @@ --------------------------------------------------------------------------- Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? - added pmsnare parser module (written by David Lang) +- bugfix: abort if imfile reads file line of more than 64KiB + Thanks to Peter Eisentraut for reporting and analysing this problem. + bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=221 --------------------------------------------------------------------------- Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 - added support for processing multi-line messages in imfile diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index ccf115c1..2b6815a4 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -156,7 +156,7 @@ rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { uchar *pNewBuf; - unsigned short iNewSize; + size_t iNewSize; DEFiRet; /* first compute the new size needed */ -- cgit v1.2.3 From 2c1021a01f75f5bb3e16b7b952a2d29494943bed Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Feb 2011 18:23:34 +0100 Subject: enhanced imfile to support non-cancel input termination --- ChangeLog | 1 + plugins/imfile/imfile.c | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 15367918..8cb94000 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ --------------------------------------------------------------------------- Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? - added pmsnare parser module (written by David Lang) +- enhanced imfile to support non-cancel input termination - bugfix: abort if imfile reads file line of more than 64KiB Thanks to Peter Eisentraut for reporting and analysing this problem. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=221 diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index c205f60e..7f6b9c24 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -213,7 +213,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) } /* loop below will be exited when strmReadLine() returns EOF */ - while(1) { + while(glbl.GetGlobalInputTermState() == 0) { CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr, pThis->readMode)); *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ @@ -289,9 +289,10 @@ BEGINrunInput int bHadFileData; /* were there at least one file with data during this run? */ CODESTARTrunInput pthread_cleanup_push(inputModuleCleanup, NULL); - while(1) { - + while(glbl.GetGlobalInputTermState() == 0) { do { + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ bHadFileData = 0; for(i = 0 ; i < iFilPtr ; ++i) { pollFile(&files[i], &bHadFileData); @@ -302,10 +303,10 @@ CODESTARTrunInput * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any * other valid scenario. So do not remove. -- rgerhards, 2008-02-14 */ - srSleep(iPollInterval, 10); - + if(glbl.GetGlobalInputTermState() == 0) + srSleep(iPollInterval, 10); } - /*NOTREACHED*/ + DBGPRINTF("imfile: terminating upon request of rsyslog core\n"); pthread_cleanup_pop(0); /* just for completeness, but never called... */ RETiRet; /* use it to make sure the housekeeping is done! */ @@ -398,6 +399,13 @@ CODESTARTafterRun ENDafterRun +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + /* The following entry points are defined in module-template.h. * In general, they need to be present, but you do NOT need to provide * any code here. @@ -416,6 +424,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt @@ -527,6 +536,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + DBGPRINTF("imfile: version %s initializing\n", VERSION); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilename", 0, eCmdHdlrGetWord, NULL, &pszFileName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfiletag", 0, eCmdHdlrGetWord, -- cgit v1.2.3 From a9736c6215cf6446c326b034648f63a79c0426f9 Mon Sep 17 00:00:00 2001 From: David Lang Date: Fri, 11 Feb 2011 07:44:09 +0100 Subject: bugfix in pmsnare when removing text from the message, decrement the length when you increment the pointer through the message. there were a number of places where this was missed. Signed-off-by: Rainer Gerhards --- plugins/pmsnare/pmsnare.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/pmsnare/pmsnare.c b/plugins/pmsnare/pmsnare.c index ce2138b2..4a9880d4 100644 --- a/plugins/pmsnare/pmsnare.c +++ b/plugins/pmsnare/pmsnare.c @@ -137,14 +137,17 @@ CODESTARTparse *p2parse = ' '; lenMsg -=(tablength-2); p2parse++; + lenMsg--; memmove(p2parse, p2parse + (tablength-2), lenMsg); *(p2parse + lenMsg) = '\n'; *(p2parse + lenMsg + 1) = '\0'; pMsg->iLenRawMsg -=(tablength-2); pMsg->iLenMSG -=(tablength-2); p2parse += snaremessage; + lenMsg -= snaremessage; *p2parse = ' '; p2parse++; + lenMsg--; lenMsg -=(tablength-2); memmove(p2parse, p2parse + (tablength-2), lenMsg); *(p2parse + lenMsg) = '\n'; @@ -178,8 +181,10 @@ CODESTARTparse } if(snaremessage) { p2parse += snaremessage; + lenMsg -= snaremessage; *p2parse = ' '; p2parse++; + lenMsg--; lenMsg -=(tablength-2); memmove(p2parse, p2parse + (tablength-2), lenMsg); *(p2parse + lenMsg) = '\n'; @@ -190,7 +195,7 @@ CODESTARTparse } } - DBGPRINTF("pmsnare: new mesage: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + DBGPRINTF("pmsnare: new message: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); finalize_it: -- cgit v1.2.3 From 1b84d5998926df6ef1b2bbfc9fe5a59a80c5564c Mon Sep 17 00:00:00 2001 From: Marius Tomaschewski Date: Fri, 11 Feb 2011 09:24:45 +0100 Subject: Improved systemd socket activation support Support for multiple unix sockets and activation in forking mode Signed-off-by: Marius Tomaschewski --- plugins/imuxsock/imuxsock.c | 89 +++++++++++++++++++++++---------------------- tools/syslogd.c | 39 ++++++++++++++++++-- 2 files changed, 81 insertions(+), 47 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 0eee1122..e456a113 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -143,7 +143,7 @@ static int startIndexUxLocalSockets; /* process fd from that index on (used to * read-only after startup */ static int nfd = 1; /* number of Unix sockets open / read-only after startup */ -static int bSysSockFromSystemd = 0; /* Did we receive the system socket from systemd? */ +static int sd_fds = 0; /* number of systemd activated sockets */ /* config settings */ static int bOmitLocalLogging = 0; @@ -372,41 +372,32 @@ openLogSocket(lstn_t *pLstn) if(pLstn->sockName[0] == '\0') return -1; - if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { - bSysSockFromSystemd = 0; /* set default */ - int r; - - /* System log socket code. Check whether an FD was passed in from systemd. If - * so, it's the /dev/log socket, so use it. */ - - r = sd_listen_fds(0); - if (r < 0) { - errmsg.LogError(-r, NO_ERRCODE, "Failed to acquire systemd socket"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - - if (r > 1) { - errmsg.LogError(EINVAL, NO_ERRCODE, "Wrong number of systemd sockets passed"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - - if (r == 1) { - pLstn->fd = SD_LISTEN_FDS_START; - r = sd_is_socket_unix(pLstn->fd, SOCK_DGRAM, -1, _PATH_LOG, 0); - if (r < 0) { - errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - - if (!r) { - errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - bSysSockFromSystemd = 1; /* indicate we got the socket from systemd */ - } else { - CHKiRet(createLogSocket(pLstn)); + pLstn->fd = -1; + + if (sd_fds > 0) { + /* Check if the current socket is a systemd activated one. + * If so, just use it. + */ + int fd; + + for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + sd_fds; fd++) { + if( sd_is_socket_unix(fd, SOCK_DGRAM, -1, pLstn->sockName, 0) == 1) { + /* ok, it matches -- just use as is */ + pLstn->fd = fd; + + dbgprintf("imuxsock: Acquired UNIX socket '%s' (fd %d) from systemd.\n", + pLstn->sockName, pLstn->fd); + break; + } + /* + * otherwise it either didn't matched *this* socket and + * we just continue to check the next one or there were + * an error and we will create a new socket bellow. + */ } - } else { + } + + if (pLstn->fd == -1) { CHKiRet(createLogSocket(pLstn)); } @@ -774,12 +765,18 @@ CODESTARTwillRun listeners[0].bUseCreds = (bWritePidSysSock || ratelimitIntervalSysSock) ? 1 : 0; listeners[0].bWritePid = bWritePidSysSock; + sd_fds = sd_listen_fds(0); + if (sd_fds < 0) { + errmsg.LogError(-r, NO_ERRCODE, "imuxsock: Failed to acquire systemd socket"); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } + /* initialize and return if will run or not */ actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfd ; i++) { if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; - dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); + dbgprintf("imuxsock: Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } } @@ -806,15 +803,19 @@ CODESTARTafterRun if (listeners[i].fd != -1) close(listeners[i].fd); - /* Clean-up files. If systemd passed us a socket it is - * systemd's job to clean it up.*/ - if(bSysSockFromSystemd) { - DBGPRINTF("imuxsock: got system socket from systemd, not unlinking it\n"); - i = 1; - } else - i = startIndexUxLocalSockets; - for(; i < nfd; i++) + /* Clean-up files. */ + for(i = startIndexUxLocalSockets; i < nfd; i++) if (listeners[i].sockName && listeners[i].fd != -1) { + + /* If systemd passed us a socket it is systemd's job to clean it up. + * Do not unlink it -- we will get same socket (node) from systemd + * e.g. on restart again. + */ + if (sd_fds > 0 && + listeners[i].fd >= SD_LISTEN_FDS_START && + listeners[i].fd < SD_LISTEN_FDS_START + sd_fds) + continue; + DBGPRINTF("imuxsock: unlinking unix socket file[%d] %s\n", i, listeners[i].sockName); unlink((char*) listeners[i].sockName); } diff --git a/tools/syslogd.c b/tools/syslogd.c index 19a5900c..52c0be5f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2429,13 +2429,46 @@ doGlblProcessInit(void) */ exit(1); /* "good" exit - after forking, not diasabling anything */ } + num_fds = getdtablesize(); close(0); /* we keep stdout and stderr open in case we have to emit something */ + i = 3; + + /* if (sd_booted()) */ { + const char *e; + char buf[24] = { '\0' }; + char *p = NULL; + unsigned long l; + int fds; + + /* fork & systemd socket activation: + * fetch listen pid and update to ours, + * when it is set to pid of our parent. + */ + if ( (e = getenv("LISTEN_PID"))) { + errno = 0; + l = strtoul(e, &p, 10); + if (errno == 0 && l > 0 && (!p || !*p)) { + if (getppid() == (pid_t)l) { + snprintf(buf, sizeof(buf), "%d", + getpid()); + setenv("LISTEN_PID", buf, 1); + } + } + } + + /* + * close only all further fds, except + * of the fds provided by systemd. + */ + sd_fds = sd_listen_fds(0); + if (sd_fds > 0) + i = SD_LISTEN_FDS_START + sd_fds; + } + for ( ; i < num_fds; i++) + (void) close(i); - if (sd_listen_fds(0) <= 0) - for (i = 3; i < num_fds; i++) - (void) close(i); untty(); } else { fputs(" Already running. If you want to run multiple instances, you need " -- cgit v1.2.3 From c90079aaf07ed574b11ade34af899cb4980ac9f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 11:25:06 +0100 Subject: fixed some compile issues in recent systemd patch --- plugins/imuxsock/imuxsock.c | 4 ++-- tools/syslogd.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index e456a113..75f97db4 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -381,7 +381,7 @@ openLogSocket(lstn_t *pLstn) int fd; for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + sd_fds; fd++) { - if( sd_is_socket_unix(fd, SOCK_DGRAM, -1, pLstn->sockName, 0) == 1) { + if( sd_is_socket_unix(fd, SOCK_DGRAM, -1, (const char*) pLstn->sockName, 0) == 1) { /* ok, it matches -- just use as is */ pLstn->fd = fd; @@ -767,7 +767,7 @@ CODESTARTwillRun sd_fds = sd_listen_fds(0); if (sd_fds < 0) { - errmsg.LogError(-r, NO_ERRCODE, "imuxsock: Failed to acquire systemd socket"); + errmsg.LogError(-sd_fds, NO_ERRCODE, "imuxsock: Failed to acquire systemd socket"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } diff --git a/tools/syslogd.c b/tools/syslogd.c index 52c0be5f..cdb31ef6 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2440,7 +2440,7 @@ doGlblProcessInit(void) char buf[24] = { '\0' }; char *p = NULL; unsigned long l; - int fds; + int sd_fds; /* fork & systemd socket activation: * fetch listen pid and update to ours, -- cgit v1.2.3 From 2d67035d1b851bdfb430e6980ca217c1807b3a9c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 11:45:13 +0100 Subject: doc: added recent changes to changelog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8cb94000..57c23cd5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,8 @@ Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? - added pmsnare parser module (written by David Lang) - enhanced imfile to support non-cancel input termination +- improved systemd socket activation thanks to Marius Tomaschweski +- bugfix: pmsnare causded abort under some conditions - bugfix: abort if imfile reads file line of more than 64KiB Thanks to Peter Eisentraut for reporting and analysing this problem. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=221 -- cgit v1.2.3 From 121f5ab4ec4e766aeae5671005325a6aef4a1806 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 14:56:02 +0100 Subject: bugfix: queue engine did not properly slow down inputs in FULL_DELAY mode... ...when in disk-assisted mode. This especially affected imfile, which created unnecessarily queue files if a large set of input file data was to process. --- ChangeLog | 4 ++++ runtime/queue.c | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 57c23cd5..6e34a436 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,6 +7,10 @@ Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? - bugfix: abort if imfile reads file line of more than 64KiB Thanks to Peter Eisentraut for reporting and analysing this problem. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=221 +- bugfix: queue engine did not properly slow down inputs in FULL_DELAY mode + when in disk-assisted mode. This especially affected imfile, which + created unnecessarily queue files if a large set of input file data was + to process. --------------------------------------------------------------------------- Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 - added support for processing multi-line messages in imfile diff --git a/runtime/queue.c b/runtime/queue.c index e4922f37..cc7c0f54 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -246,6 +246,7 @@ qqueueAdviseMaxWorkers(qqueue_t *pThis) if(!pThis->bEnqOnly) { if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) { + DBGOPRINT((obj_t*) pThis, "(re)activating DA worker\n"); wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } else { if(getLogicalQueueSize(pThis) == 0) { @@ -1211,7 +1212,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread /* set some water marks so that we have useful defaults if none are set specifically */ pThis->iFullDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 3; /* default 97% */ pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */ - pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; @@ -1819,6 +1819,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; uchar pszBuf[64]; + int wrk; uchar *qName; size_t lenBuf; @@ -1850,6 +1851,16 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* call type-specific constructor */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ + /* re-adjust some params if required */ + if(pThis->bIsDA) { + /* if we are in DA mode, we must make sure full delayable messages do not + * initiate going to disk! + */ + wrk = pThis->iHighWtrMrk - (pThis->iHighWtrMrk / 100) * 50; /* 50% of high water mark */ + if(wrk < pThis->iFullDlyMrk) + pThis->iFullDlyMrk = wrk; + } + DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, " "full delay %d, light delay %d, deq batch size %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, -- cgit v1.2.3 From 6a18d25cbec2676a7910ff038170716293abe89f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 17:06:20 +0100 Subject: removed no longer needed code --- runtime/queue.c | 2 -- runtime/queue.h | 1 - 2 files changed, 3 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index cc7c0f54..9f63a338 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1842,7 +1842,6 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ } pthread_mutex_init(&pThis->mutThrdMgmt, NULL); - pthread_cond_init (&pThis->condDAReady, NULL); pthread_cond_init (&pThis->notFull, NULL); pthread_cond_init (&pThis->notEmpty, NULL); pthread_cond_init (&pThis->belowFullDlyWtrMrk, NULL); @@ -2140,7 +2139,6 @@ CODESTARTobjDestruct(qqueue) free(pThis->mut); } pthread_mutex_destroy(&pThis->mutThrdMgmt); - pthread_cond_destroy(&pThis->condDAReady); pthread_cond_destroy(&pThis->notFull); pthread_cond_destroy(&pThis->notEmpty); pthread_cond_destroy(&pThis->belowFullDlyWtrMrk); diff --git a/runtime/queue.h b/runtime/queue.h index 38e248cd..97057180 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -124,7 +124,6 @@ struct queue_s { pthread_cond_t notFull, notEmpty; pthread_cond_t belowFullDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t belowLightDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ - pthread_cond_t condDAReady;/* signalled when the DA queue is fully initialized and ready for processing */ int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* the following variables are always present, because they -- cgit v1.2.3 From f3d354da3e373f9c4890a78e5274a6ba02f1c8cb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 17:47:30 +0100 Subject: bugfix: very long running actions could prevent shutdown under some circumstances This has now been solved, at least for common situations. --- ChangeLog | 3 +++ plugins/imfile/imfile.c | 6 +++--- runtime/queue.c | 2 +- threads.c | 6 ++++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6e34a436..7d2bbc20 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,9 @@ Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? when in disk-assisted mode. This especially affected imfile, which created unnecessarily queue files if a large set of input file data was to process. +- bugfix: very long running actions could prevent shutdown under some + circumstances. This has now been solved, at least for common + situations. --------------------------------------------------------------------------- Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 - added support for processing multi-line messages in imfile diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 7f6b9c24..c71e425e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -291,13 +291,13 @@ CODESTARTrunInput pthread_cleanup_push(inputModuleCleanup, NULL); while(glbl.GetGlobalInputTermState() == 0) { do { - if(glbl.GetGlobalInputTermState() == 1) - break; /* terminate input! */ bHadFileData = 0; for(i = 0 ; i < iFilPtr ; ++i) { + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ pollFile(&files[i], &bHadFileData); } - } while(iFilPtr > 1 && bHadFileData == 1); /* warning: do...while()! */ + } while(iFilPtr > 1 && bHadFileData == 1 && glbl.GetGlobalInputTermState() == 0); /* warning: do...while()! */ /* Note: the additional 10ns wait is vitally important. It guards rsyslog against totally * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any diff --git a/runtime/queue.c b/runtime/queue.c index 9f63a338..76327f6a 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1497,7 +1497,7 @@ DequeueConsumable(qqueue_t *pThis, wti_t *pWti) * now that we dequeue batches of pointers, this is much less an issue... * rgerhards, 2009-04-22 */ - if(iQueueSize < pThis->iFullDlyMrk / 2) { + if(iQueueSize < pThis->iFullDlyMrk / 2 || glbl.GetGlobalInputTermState() == 1) { pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk); } diff --git a/threads.c b/threads.c index d4e14527..fcafce4b 100644 --- a/threads.c +++ b/threads.c @@ -101,12 +101,14 @@ thrdTerminateNonCancel(thrdInfo_t *pThis) do { d_pthread_mutex_lock(&pThis->mutThrd); pthread_kill(pThis->thrdID, SIGTTIN); - timeoutComp(&tTimeout, 10); /* a fixed 10ms timeout, do after lock (may take long!) */ + timeoutComp(&tTimeout, 1000); /* a fixed 1sec timeout */ ret = d_pthread_cond_timedwait(&pThis->condThrdTerm, &pThis->mutThrd, &tTimeout); d_pthread_mutex_unlock(&pThis->mutThrd); if(Debug) { if(ret == ETIMEDOUT) { - dbgprintf("input thread term: had a timeout waiting on thread termination\n"); + dbgprintf("input thread term: timeout expired waiting on thread termination - canceling\n"); + pthread_cancel(pThis->thrdID); + pThis->bIsActive = 0; } else if(ret == 0) { dbgprintf("input thread term: thread returned normally and is terminated\n"); } else { -- cgit v1.2.3 From 49c2bc380c34c346dafffe5c5a4059fa2f604680 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 15 Feb 2011 11:06:15 +0100 Subject: improved error reporting for $WorkDirectory non-existance and other detectable problems are now reported, and the work directory is NOT set in this case --- ChangeLog | 3 +++ runtime/glbl.c | 37 ++++++++++++++++++++++++++++++++++++- runtime/rsyslog.h | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 7d2bbc20..2a22f5b2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,9 @@ Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? - added pmsnare parser module (written by David Lang) - enhanced imfile to support non-cancel input termination - improved systemd socket activation thanks to Marius Tomaschweski +- improved error reporting for $WorkDirectory + non-existance and other detectable problems are now reported, + and the work directory is NOT set in this case - bugfix: pmsnare causded abort under some conditions - bugfix: abort if imfile reads file line of more than 64KiB Thanks to Peter Eisentraut for reporting and analysing this problem. diff --git a/runtime/glbl.c b/runtime/glbl.c index c7bb88a5..68eb276c 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -31,6 +31,9 @@ #include "config.h" #include #include +#include +#include +#include #include #include "rsyslog.h" @@ -40,6 +43,7 @@ #include "glbl.h" #include "prop.h" #include "atomic.h" +#include "errmsg.h" /* some defaults */ #ifndef DFLT_NETSTRM_DRVR @@ -49,6 +53,7 @@ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(prop) +DEFobjCurrIf(errmsg) /* static data * For this object, these variables are obviously what makes the "meat" of the @@ -147,6 +152,35 @@ static void SetGlobalInputTermination(void) } +/* This function is used to set the global work directory name. + * It verifies that the provided directory actually exists and + * emits an error message if not. + * rgerhards, 2011-02-16 + */ +static rsRetVal setWorkDir(void __attribute__((unused)) *pVal, uchar *pNewVal) +{ + DEFiRet; + struct stat sb; + + if(stat((char*) pNewVal, &sb) != 0) { + errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: %s can not be " + "accessed, probably does not exist - directive ignored", pNewVal); + ABORT_FINALIZE(RS_RET_ERR_WRKDIR); + } + + if(!S_ISDIR(sb.st_mode)) { + errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: %s not a directory - directive ignored", + pNewVal); + ABORT_FINALIZE(RS_RET_ERR_WRKDIR); + } + + free(pszWorkDir); + pszWorkDir = pNewVal; + +finalize_it: + RETiRet; +} + /* return our local hostname. if it is not set, "[localhost]" is returned */ static uchar* @@ -354,9 +388,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* register config handlers (TODO: we need to implement a way to unregister them) */ - CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, NULL, &pszWorkDir, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, setWorkDir, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"dropmsgswithmaliciousdnsptrrecords", 0, eCmdHdlrBinary, NULL, &bDropMalPTRMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 0402f159..78e61bcb 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -340,6 +340,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_HDFS_WRITE = -2178, /**< error writing to HDFS */ RS_RET_ERR_HDFS_OPEN = -2179, /**< error during hdfsOpen (e.g. file does not exist) */ RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */ + RS_RET_ERR_WRKDIR = -2181, /**< problems with the rsyslog working directory */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 772e0ae8460478e43caac55dfa3182dd4b58b52a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 16 Feb 2011 11:46:56 +0100 Subject: cleanup: no longer parameter in iminternal system removed --- tools/iminternal.c | 6 ++---- tools/iminternal.h | 5 ++--- tools/syslogd.c | 5 ++--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tools/iminternal.c b/tools/iminternal.c index bd1fa128..12534ba0 100644 --- a/tools/iminternal.c +++ b/tools/iminternal.c @@ -89,7 +89,7 @@ finalize_it: * The interface of this function is modelled after syslogd/logmsg(), * for which it is an "replacement". */ -rsRetVal iminternalAddMsg(int pri, msg_t *pMsg) +rsRetVal iminternalAddMsg(msg_t *pMsg) { DEFiRet; iminternal_t *pThis; @@ -98,7 +98,6 @@ rsRetVal iminternalAddMsg(int pri, msg_t *pMsg) CHKiRet(iminternalConstruct(&pThis)); - pThis->pri = pri; pThis->pMsg = pMsg; CHKiRet(llAppend(&llMsgs, NULL, (void*) pThis)); @@ -118,7 +117,7 @@ finalize_it: * from the list and return it to the caller. The caller is * responsible for freeing the message! */ -rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg) +rsRetVal iminternalRemoveMsg(msg_t **ppMsg) { DEFiRet; iminternal_t *pThis; @@ -128,7 +127,6 @@ rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg) assert(ppMsg != NULL); CHKiRet(llGetNextElt(&llMsgs, &llCookie, (void*)&pThis)); - *pPri = pThis->pri; *ppMsg = pThis->pMsg; pThis->pMsg = NULL; /* we do no longer own it - important for destructor */ diff --git a/tools/iminternal.h b/tools/iminternal.h index f1062a15..8a9e2506 100644 --- a/tools/iminternal.h +++ b/tools/iminternal.h @@ -33,7 +33,6 @@ * The short name is cslch (Configfile SysLine CommandHandler) */ struct iminternal_s { /* config file sysline parse entry */ - int pri; msg_t *pMsg; /* the message (in all its glory) */ }; typedef struct iminternal_s iminternal_t; @@ -41,8 +40,8 @@ typedef struct iminternal_s iminternal_t; /* prototypes */ rsRetVal modInitIminternal(void); rsRetVal modExitIminternal(void); -rsRetVal iminternalAddMsg(int pri, msg_t *pMsg); +rsRetVal iminternalAddMsg(msg_t *pMsg); rsRetVal iminternalHaveMsgReady(int* pbHaveOne); -rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg); +rsRetVal iminternalRemoveMsg(msg_t **ppMsg); #endif /* #ifndef IMINTERNAL_H_INCLUDED */ diff --git a/tools/syslogd.c b/tools/syslogd.c index cdb31ef6..c4a3b270 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -574,7 +574,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) } if(bHaveMainQueue == 0) { /* not yet in queued mode */ - iminternalAddMsg(pri, pMsg); + iminternalAddMsg(pMsg); } else { /* we have the queue, so we can simply provide the * message to the queue engine. @@ -1868,10 +1868,9 @@ void sigttin_handler() */ static inline void processImInternal(void) { - int iPri; msg_t *pMsg; - while(iminternalRemoveMsg(&iPri, &pMsg) == RS_RET_OK) { + while(iminternalRemoveMsg(&pMsg) == RS_RET_OK) { submitMsg(pMsg); } } -- cgit v1.2.3 From 7052d3c378ff600f9f03a05b1918d8eda6b76e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dra=C5=BEen=20Ka=C4=8Dar?= Date: Wed, 16 Feb 2011 18:20:59 +0100 Subject: bugfix: fixed compile problem due to empty structs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this occured only on some platforms/compilers. thanks to Dražen Kačar for the fix Signed-off-by: Rainer Gerhards --- ChangeLog | 3 +++ runtime/datetime.h | 1 + runtime/errmsg.h | 1 + runtime/expr.h | 1 + runtime/modules.h | 1 + tools/omdiscard.c | 1 + 6 files changed, 8 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2a22f5b2..7fba4e8d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,9 @@ Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? - bugfix: very long running actions could prevent shutdown under some circumstances. This has now been solved, at least for common situations. +- bugfix: fixed compile problem due to empty structs + this occured only on some platforms/compilers. thanks to Dražen Kačar + for the fix --------------------------------------------------------------------------- Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 - added support for processing multi-line messages in imfile diff --git a/runtime/datetime.h b/runtime/datetime.h index 82bd415b..70bbf416 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -28,6 +28,7 @@ /* the datetime object */ typedef struct datetime_s { + char dummy; } datetime_t; diff --git a/runtime/errmsg.h b/runtime/errmsg.h index 799954fb..ac7018b3 100644 --- a/runtime/errmsg.h +++ b/runtime/errmsg.h @@ -30,6 +30,7 @@ /* the errmsg object */ typedef struct errmsg_s { + char dummy; } errmsg_t; diff --git a/runtime/expr.h b/runtime/expr.h index 974b71ec..1afe1a1f 100644 --- a/runtime/expr.h +++ b/runtime/expr.h @@ -30,6 +30,7 @@ /* a node inside an expression tree */ typedef struct exprNode_s { + char dummy; } exprNode_t; diff --git a/runtime/modules.h b/runtime/modules.h index 49586e8d..df1afbc3 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -119,6 +119,7 @@ struct modInfo_s { rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); } om; struct { /* data for library modules */ + char dummy; } lm; struct { /* data for parser modules */ rsRetVal (*parse)(msg_t*); diff --git a/tools/omdiscard.c b/tools/omdiscard.c index f13144e8..227ad8e2 100644 --- a/tools/omdiscard.c +++ b/tools/omdiscard.c @@ -44,6 +44,7 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA typedef struct _instanceData { + char dummy; } instanceData; /* we do not need a createInstance()! -- cgit v1.2.3 From c550907139b9a13e1afd771f94f46a153fde08a2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Feb 2011 14:25:43 +0100 Subject: preparing for 5.7.4 release --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7fba4e8d..fc2c3c97 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-?? +Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-17 - added pmsnare parser module (written by David Lang) - enhanced imfile to support non-cancel input termination - improved systemd socket activation thanks to Marius Tomaschweski diff --git a/configure.ac b/configure.ac index 8eb840b0..1db9f26e 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.3],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.4],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index e8a1ccfa..4be37fd5 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.3 (beta branch) of rsyslog. +

    This documentation is for version 5.7.4 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 39406781e51c7305cb9f35f5f9fed9c10dea9ecd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Feb 2011 15:12:54 +0100 Subject: bugfix: minor race condition in action.c - considered cosmetic This is considered cosmetic as multiple threads tried to write exactly the same value into the same memory location without sync. The method has been changed so this can no longer happen. --- ChangeLog | 6 ++++++ action.c | 25 +++++++++++++------------ action.h | 1 - threads.c | 2 +- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index fc2c3c97..a78e9e62 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ --------------------------------------------------------------------------- +Version 5.7.5 [V5-BETA] (rgerhards), 2011-02-?? +- bugfix: minor race condition in action.c - considered cosmetic + This is considered cosmetic as multiple threads tried to write exactly + the same value into the same memory location without sync. The method + has been changed so this can no longer happen. +--------------------------------------------------------------------------- Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-17 - added pmsnare parser module (written by David Lang) - enhanced imfile to support non-cancel input termination diff --git a/action.c b/action.c index 4bf8ba04..36830577 100644 --- a/action.c +++ b/action.c @@ -494,7 +494,8 @@ static inline void actionSuspend(action_t *pThis, time_t ttNow) * kind of facility: in the first place, the module should return a proper indication * of its inability to recover. -- rgerhards, 2010-04-26. */ -static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) +static inline rsRetVal +actionDoRetry(action_t *pThis, time_t ttNow, int *pbShutdownImmediate) { int iRetries; int iSleepPeriod; @@ -504,7 +505,7 @@ static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) ASSERT(pThis != NULL); iRetries = 0; - while((*pThis->pbShutdownImmediate == 0) && pThis->eState == ACT_STATE_RTRY) { + while((*pbShutdownImmediate == 0) && pThis->eState == ACT_STATE_RTRY) { iRet = pThis->pMod->tryResume(pThis->pModData); if((pThis->iResumeOKinRow > 999) && (pThis->iResumeOKinRow % 1000 == 0)) { bTreatOKasSusp = 1; @@ -524,7 +525,7 @@ static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) iSleepPeriod = pThis->iResumeInterval; ttNow += iSleepPeriod; /* not truly exact, but sufficiently... */ srSleep(iSleepPeriod, 0); - if(*pThis->pbShutdownImmediate) { + if(*pbShutdownImmediate) { ABORT_FINALIZE(RS_RET_FORCE_TERM); } } @@ -545,7 +546,7 @@ finalize_it: /* try to resume an action -- rgerhards, 2007-08-02 * changed to new action state engine -- rgerhards, 2009-05-07 */ -static rsRetVal actionTryResume(action_t *pThis) +static rsRetVal actionTryResume(action_t *pThis, int *pbShutdownImmediate) { DEFiRet; time_t ttNow = NO_TIME_PROVIDED; @@ -569,7 +570,7 @@ static rsRetVal actionTryResume(action_t *pThis) if(pThis->eState == ACT_STATE_RTRY) { if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */ datetime.GetTime(&ttNow); - CHKiRet(actionDoRetry(pThis, ttNow)); + CHKiRet(actionDoRetry(pThis, ttNow, pbShutdownImmediate)); } if(Debug && (pThis->eState == ACT_STATE_RTRY ||pThis->eState == ACT_STATE_SUSP)) { @@ -586,12 +587,12 @@ finalize_it: * depending on its current state. * rgerhards, 2009-05-07 */ -static inline rsRetVal actionPrepare(action_t *pThis) +static inline rsRetVal actionPrepare(action_t *pThis, int *pbShutdownImmediate) { DEFiRet; assert(pThis != NULL); - CHKiRet(actionTryResume(pThis)); + CHKiRet(actionTryResume(pThis, pbShutdownImmediate)); /* if we are now ready, we initialize the transaction and advance * action state accordingly @@ -800,14 +801,14 @@ finalize_it: * rgerhards, 2008-01-28 */ static inline rsRetVal -actionProcessMessage(action_t *pThis, msg_t *pMsg, void *actParams) +actionProcessMessage(action_t *pThis, msg_t *pMsg, void *actParams, int *pbShutdownImmediate) { DEFiRet; ASSERT(pThis != NULL); ISOBJ_TYPE_assert(pMsg, msg); - CHKiRet(actionPrepare(pThis)); + CHKiRet(actionPrepare(pThis, pbShutdownImmediate)); if(pThis->eState == ACT_STATE_ITX) CHKiRet(actionCallDoAction(pThis, pMsg, actParams)); @@ -834,7 +835,7 @@ finishBatch(action_t *pThis, batch_t *pBatch) FINALIZE; /* nothing to do */ } - CHKiRet(actionPrepare(pThis)); + CHKiRet(actionPrepare(pThis, pBatch->pbShutdownImmediate)); if(pThis->eState == ACT_STATE_ITX) { iRet = pThis->pMod->mod.om.endTransaction(pThis->pModData); switch(iRet) { @@ -901,7 +902,8 @@ tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem) && pBatch->pElem[i].state != BATCH_STATE_DISC && ((pAction->bExecWhenPrevSusp == 0) || pBatch->pElem[i].bPrevWasSuspended) ) { pMsg = (msg_t*) pBatch->pElem[i].pUsrp; - localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams); + localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams, + pBatch->pbShutdownImmediate); DBGPRINTF("action call returned %d\n", localRet); /* Note: we directly modify the batch object state, because we know that * wo do not overwrite BATCH_STATE_DISC indicators! @@ -1072,7 +1074,6 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) pbShutdownImmdtSave = pBatch->pbShutdownImmediate; pBatch->pbShutdownImmediate = pbShutdownImmediate; - pAction->pbShutdownImmediate = pBatch->pbShutdownImmediate; CHKiRet(prepareBatch(pAction, pBatch)); /* We now must guard the output module against execution by multiple threads. The diff --git a/action.h b/action.h index 0c86ef88..e57a0acf 100644 --- a/action.h +++ b/action.h @@ -88,7 +88,6 @@ struct action_s { SYNC_OBJ_TOOL; /* required for mutex support */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ uchar *pszName; /* action name (for documentation) */ - int *pbShutdownImmediate;/* to facilitate shutdown, if var is 1, shut down immediately */ DEF_ATOMIC_HELPER_MUT(mutCAS); }; diff --git a/threads.c b/threads.c index fcafce4b..11c9a1d8 100644 --- a/threads.c +++ b/threads.c @@ -196,8 +196,8 @@ static void* thrdStarter(void *arg) * keep the thread debugger happer, it would not really be necessary with * the logic we employ...) */ - pThis->bIsActive = 0; d_pthread_mutex_lock(&pThis->mutThrd); + pThis->bIsActive = 0; pthread_cond_signal(&pThis->condThrdTerm); d_pthread_mutex_unlock(&pThis->mutThrd); -- cgit v1.2.3 From 2f1ef6164afa3a91f0bdd92507d0e8d557bea555 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 18 Feb 2011 16:04:50 +0100 Subject: enhance: imfile did not yet support multiple rulesets, now added we do this directly in the beta because a) it does not affect existing functionality and b) one may argue that this missing functionality is close to a bug. --- ChangeLog | 4 ++++ doc/imfile.html | 5 ++++- plugins/imfile/imfile.c | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 057a1b7c..d502ce35 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- Version 5.7.5 [V5-BETA] (rgerhards), 2011-02-?? +- enhance: imfile did not yet support multiple rulesets, now added + we do this directly in the beta because a) it does not affect existing + functionality and b) one may argue that this missing functionality is + close to a bug. - bugfix: minor race condition in action.c - considered cosmetic This is considered cosmetic as multiple threads tried to write exactly the same value into the same memory location without sync. The method diff --git a/doc/imfile.html b/doc/imfile.html index 66c13e06..60726ceb 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -97,10 +97,13 @@ to fatal errors (like power fail). Note that this setting affects imfile performance, especially when set to a low value. Frequently writing the state file is very time consuming.

  • $InputFileReadMode [mode]
    -Available in 5.7.3+ +Available in 5.7.5+
    Mode to be used when reading lines. 0 (the default) means that each line is forwarded as its own log message. +
  • $InputFileBindRuleset <ruleset>
    +Available in 5.7.5+, 6.1.5+ +Binds the listener to a specific ruleset.
    • Caveats/Known Bugs:

    So far, only 100 files can be monitored. If more are needed, diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index c71e425e..1dd5e65c 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -48,6 +48,7 @@ #include "unicode-helper.h" #include "prop.h" #include "stringbuf.h" +#include "ruleset.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -60,6 +61,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) DEFobjCurrIf(strm) DEFobjCurrIf(prop) +DEFobjCurrIf(ruleset) typedef struct fileInfo_s { uchar *pszFileName; @@ -72,6 +74,7 @@ typedef struct fileInfo_s { int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ int readMode; /* which mode to use in ReadMulteLine call? */ + ruleset_t *pRuleset; /* ruleset to bind listener to (use system default if unspecified) */ } fileInfo_t; @@ -87,6 +90,7 @@ static int iPersistStateInterval = 0; /* how often if state file to be persisted static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ static int readMode = 0; /* mode to use for ReadMultiLine call */ +static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ static int iFilPtr = 0; /* number of files to be monitored; pointer to next free spot during config */ #define MAX_INPUT_FILES 100 @@ -116,6 +120,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); + MsgSetRuleset(pMsg, pInfo->pRuleset); CHKiRet(submitMsg(pMsg)); finalize_it: RETiRet; @@ -418,6 +423,7 @@ CODESTARTmodExit objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDmodExit @@ -459,6 +465,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iFacility = 128; /* local0 */ iSeverity = 5; /* notice, as of rfc 3164 */ readMode = 0; + pBindRuleset = NULL; RETiRet; } @@ -502,6 +509,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->iPersistStateInterval = iPersistStateInterval; pThis->nRecords = 0; pThis->readMode = readMode; + pThis->pRuleset = pBindRuleset; iPersistStateInterval = 0; } else { errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one"); @@ -517,6 +525,29 @@ finalize_it: RETiRet; } + +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal +setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imfile current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + /* modInit() is called once the module is loaded. It must perform all module-wide * initialization tasks. There are also a number of housekeeping tasks that the * framework requires. These are handled by the macros. Please note that the @@ -534,6 +565,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); DBGPRINTF("imfile: version %s initializing\n", VERSION); @@ -553,6 +585,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &readMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilebindruleset", 0, eCmdHdlrGetWord, + setRuleset, NULL, STD_LOADABLE_MODULE_ID)); /* that command ads a new file! */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrunfilemonitor", 0, eCmdHdlrGetWord, addMonitor, NULL, STD_LOADABLE_MODULE_ID)); -- cgit v1.2.3 From 4d35185317fd03e2a820013e7ab3ca5c760db670 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Feb 2011 08:14:01 +0100 Subject: regression: fixed compile error with --enable-debug --- tools/iminternal.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/iminternal.c b/tools/iminternal.c index 12534ba0..167e2b29 100644 --- a/tools/iminternal.c +++ b/tools/iminternal.c @@ -123,7 +123,6 @@ rsRetVal iminternalRemoveMsg(msg_t **ppMsg) iminternal_t *pThis; linkedListCookie_t llCookie = NULL; - assert(pPri != NULL); assert(ppMsg != NULL); CHKiRet(llGetNextElt(&llMsgs, &llCookie, (void*)&pThis)); -- cgit v1.2.3 From 026f59388a105597c3c6890bb0c83652cb712903 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Feb 2011 09:10:45 +0100 Subject: bugfix: imuxsock did no longer remove trailing LF This was a regression from the imuxsock partial rewrite. Happened because the message is no longer run through the standard parsers. Now imuxsock does this sanitization itself. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=224 --- ChangeLog | 5 +++++ plugins/imuxsock/imuxsock.c | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index d502ce35..71e6f3ed 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,11 @@ Version 5.7.5 [V5-BETA] (rgerhards), 2011-02-?? we do this directly in the beta because a) it does not affect existing functionality and b) one may argue that this missing functionality is close to a bug. +- bugfix: imuxsock did no longer remove trailing LF + This was a regression from the imuxsock partial rewrite. Happened + because the message is no longer run through the standard parsers. + Now imuxsock does this sanitization itself. + bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=224 - bugfix: minor race condition in action.c - considered cosmetic This is considered cosmetic as multiple threads tried to write exactly the same value into the same memory location without sync. The method diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 75f97db4..136e4196 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -517,6 +517,10 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) * rate-limiting as well. */ parse = pRcv; + if(pRcv[lenRcv-1] = '\n') { + DBGPRINTF("imuxsock: found trailing LF, removing\n"); + lenRcv--; + } lenMsg = lenRcv; parse++; lenMsg--; /* '<' */ -- cgit v1.2.3 From 314d3532109cca574aabc3e960586f499966e868 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Feb 2011 11:41:50 +0100 Subject: fixed dumb but important typo in code... thanks to Michael Biebl for alterting me --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 136e4196..0b5ca6be 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -517,7 +517,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) * rate-limiting as well. */ parse = pRcv; - if(pRcv[lenRcv-1] = '\n') { + if(pRcv[lenRcv-1] == '\n') { DBGPRINTF("imuxsock: found trailing LF, removing\n"); lenRcv--; } -- cgit v1.2.3 From 36c1649fc187090bfd5af2c48bb44ce25a0fdef6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Feb 2011 14:08:37 +0100 Subject: improved testbench, added tests for imuxsock --- ChangeLog | 1 + tests/Makefile.am | 13 ++++++++++++- tests/imuxsock_logger_root.sh | 18 ++++++++++++++++++ tests/imuxsock_traillf_root.sh | 18 ++++++++++++++++++ tests/resultdata/imuxsock_logger.log | 1 + tests/resultdata/imuxsock_traillf.log | 1 + tests/syslog_lf.c | 9 +++++++++ tests/testsuites/imuxsock_logger_root.conf | 7 +++++++ tests/testsuites/imuxsock_traillf_root.conf | 7 +++++++ 9 files changed, 74 insertions(+), 1 deletion(-) create mode 100755 tests/imuxsock_logger_root.sh create mode 100755 tests/imuxsock_traillf_root.sh create mode 100644 tests/resultdata/imuxsock_logger.log create mode 100644 tests/resultdata/imuxsock_traillf.log create mode 100644 tests/syslog_lf.c create mode 100644 tests/testsuites/imuxsock_logger_root.conf create mode 100644 tests/testsuites/imuxsock_traillf_root.conf diff --git a/ChangeLog b/ChangeLog index 71e6f3ed..82dd5ac1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,7 @@ Version 5.7.5 [V5-BETA] (rgerhards), 2011-02-?? we do this directly in the beta because a) it does not affect existing functionality and b) one may argue that this missing functionality is close to a bug. +- improved testbench, added tests for imuxsock - bugfix: imuxsock did no longer remove trailing LF This was a regression from the imuxsock partial rewrite. Happened because the message is no longer run through the standard parsers. diff --git a/tests/Makefile.am b/tests/Makefile.am index 25d972d7..0ea66b0f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller syslog_lf TESTS = $(TESTRUNS) cfg.sh \ arrayqueue.sh \ linkedlistqueue.sh \ @@ -46,6 +46,8 @@ TESTS = $(TESTRUNS) cfg.sh \ pipe_noreader.sh \ dircreate_dflt.sh \ dircreate_off.sh \ + imuxsock_logger_root.sh \ + imuxsock_traillf_root.sh \ queue-persist.sh if ENABLE_IMPTCP @@ -316,6 +318,12 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/dircreate_dflt.conf \ dircreate_off.sh \ testsuites/dircreate_off.conf \ + imuxsock_logger_root.sh \ + testsuites/imuxsock_logger_root.conf \ + resultdata/imuxsock_logger.log \ + imuxsock_traillf_root.sh \ + testsuites/imuxsock_traillf_root.conf \ + resultdata/imuxsock_traillf.log \ cfg.sh ourtail_SOURCES = ourtail.c @@ -331,6 +339,9 @@ tcpflood_LDADD = $(SOL_LIBS) syslog_caller_SOURCES = syslog_caller.c syslog_caller_LDADD = $(SOL_LIBS) +syslog_lf_SOURCES = syslog_lf.c +syslog_lf_LDADD = $(SOL_LIBS) + diagtalker_SOURCES = diagtalker.c diagtalker_LDADD = $(SOL_LIBS) diff --git a/tests/imuxsock_logger_root.sh b/tests/imuxsock_logger_root.sh new file mode 100755 index 00000000..e94a83fe --- /dev/null +++ b/tests/imuxsock_logger_root.sh @@ -0,0 +1,18 @@ +# note: we must be root and no other syslogd running in order to +# carry out this test. +echo \[imuxsock_logger_root.sh\]: test trailing LF handling in imuxsock +echo This test must be run as root with no other active syslogd +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imuxsock_logger_root.conf +# send a message with trailing LF +logger test +# the sleep below is needed to prevent too-early termination of rsyslogd +$srcdir/msleep 100 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! +cmp rsyslog.out.log $srcdir/resultdata/imuxsock_logger.log +if [ ! $? -eq 0 ]; then +echo "imuxsock_logger.sh failed" +exit 1 +fi; +source $srcdir/diag.sh exit diff --git a/tests/imuxsock_traillf_root.sh b/tests/imuxsock_traillf_root.sh new file mode 100755 index 00000000..c838e0bd --- /dev/null +++ b/tests/imuxsock_traillf_root.sh @@ -0,0 +1,18 @@ +# note: we must be root and no other syslogd running in order to +# carry out this test +echo \[imuxsock_traillf_root.sh\]: test trailing LF handling in imuxsock +echo This test must be run as root with no other active syslogd +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imuxsock_traillf_root.conf +# send a message with trailing LF +$srcdir/syslog_lf +# the sleep below is needed to prevent too-early termination of rsyslogd +$srcdir/msleep 100 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! +cmp rsyslog.out.log $srcdir/resultdata/imuxsock_traillf.log +if [ ! $? -eq 0 ]; then +echo "imuxsock_traillf_root.sh failed" +exit 1 +fi; +source $srcdir/diag.sh exit diff --git a/tests/resultdata/imuxsock_logger.log b/tests/resultdata/imuxsock_logger.log new file mode 100644 index 00000000..883dabdf --- /dev/null +++ b/tests/resultdata/imuxsock_logger.log @@ -0,0 +1 @@ + test diff --git a/tests/resultdata/imuxsock_traillf.log b/tests/resultdata/imuxsock_traillf.log new file mode 100644 index 00000000..883dabdf --- /dev/null +++ b/tests/resultdata/imuxsock_traillf.log @@ -0,0 +1 @@ + test diff --git a/tests/syslog_lf.c b/tests/syslog_lf.c new file mode 100644 index 00000000..a140244e --- /dev/null +++ b/tests/syslog_lf.c @@ -0,0 +1,9 @@ +/* This tool deliberately logs a message with the a trailing LF */ +#include +#include + +int main() +{ + syslog(LOG_NOTICE, "test\n"); + return 0; +} diff --git a/tests/testsuites/imuxsock_logger_root.conf b/tests/testsuites/imuxsock_logger_root.conf new file mode 100644 index 00000000..8336ecfa --- /dev/null +++ b/tests/testsuites/imuxsock_logger_root.conf @@ -0,0 +1,7 @@ +# rgerhards, 2011-02-21 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imuxsock/.libs/imuxsock + +$template outfmt,"%msg:%\n" +*.notice ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/imuxsock_traillf_root.conf b/tests/testsuites/imuxsock_traillf_root.conf new file mode 100644 index 00000000..8336ecfa --- /dev/null +++ b/tests/testsuites/imuxsock_traillf_root.conf @@ -0,0 +1,7 @@ +# rgerhards, 2011-02-21 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imuxsock/.libs/imuxsock + +$template outfmt,"%msg:%\n" +*.notice ./rsyslog.out.log;outfmt -- cgit v1.2.3 From 70bd55d51b6107231b841bfa03d5620bf984e3f2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Feb 2011 11:55:12 +0100 Subject: bugfix: imuxsock does not sanitization at all (regression) This also causes problems when NUL characters are present inside the message (as is the case with spamd). --- ChangeLog | 3 +-- plugins/imuxsock/imuxsock.c | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 82dd5ac1..f771ad54 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,10 +5,9 @@ Version 5.7.5 [V5-BETA] (rgerhards), 2011-02-?? functionality and b) one may argue that this missing functionality is close to a bug. - improved testbench, added tests for imuxsock -- bugfix: imuxsock did no longer remove trailing LF +- bugfix: imuxsock did no longer sanitize received messages This was a regression from the imuxsock partial rewrite. Happened because the message is no longer run through the standard parsers. - Now imuxsock does this sanitization itself. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=224 - bugfix: minor race condition in action.c - considered cosmetic This is considered cosmetic as multiple threads tried to write exactly diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 0b5ca6be..9f1e51b7 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -46,6 +46,7 @@ #include "net.h" #include "glbl.h" #include "msg.h" +#include "parser.h" #include "prop.h" #include "debug.h" #include "unlimited_select.h" @@ -81,6 +82,7 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(parser) DEFobjCurrIf(datetime) DEFobjCurrIf(statsobj) @@ -517,10 +519,6 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) * rate-limiting as well. */ parse = pRcv; - if(pRcv[lenRcv-1] == '\n') { - DBGPRINTF("imuxsock: found trailing LF, removing\n"); - lenRcv--; - } lenMsg = lenRcv; parse++; lenMsg--; /* '<' */ @@ -547,6 +545,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); + parser.SanitizeMsg(pMsg); MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); @@ -840,6 +839,7 @@ BEGINmodExit CODESTARTmodExit statsobj.Destruct(&modStats); + objRelease(parser, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); @@ -901,6 +901,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(statsobj, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); -- cgit v1.2.3 From b7ee1de6b0dbdc67bbb239f44719fb4a50054fb5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Feb 2011 15:35:52 +0100 Subject: the last fix introduced another regression, fixed now The previous fix left variable lenMsg in an inconsistent state when sanitization actually happend. This could lead to message truncation. --- plugins/imuxsock/imuxsock.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 9f1e51b7..ff38852c 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -503,6 +503,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) { msg_t *pMsg; int lenMsg; + int offs; int i; uchar *parse; int pri; @@ -520,13 +521,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) */ parse = pRcv; lenMsg = lenRcv; + offs = 1; /* '<' */ - parse++; lenMsg--; /* '<' */ + parse++; pri = 0; - while(lenMsg && isdigit(*parse)) { + while(offs < lenMsg && isdigit(*parse)) { pri = pri * 10 + *parse - '0'; ++parse; - --lenMsg; + ++offs; } facil = LOG_FAC(pri); sever = LOG_PRI(pri); @@ -546,12 +548,13 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); parser.SanitizeMsg(pMsg); + lenMsg = pMsg->iLenRawMsg - offs; MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); pMsg->iFacility = facil; pMsg->iSeverity = sever; - MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); + MsgSetAfterPRIOffs(pMsg, offs); parse++; lenMsg--; /* '>' */ @@ -571,7 +574,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) fixPID(bufParseTAG, &i, cred); MsgSetTAG(pMsg, bufParseTAG, i); - MsgSetMSGoffs(pMsg, lenRcv - lenMsg); + MsgSetMSGoffs(pMsg, pMsg->iLenRawMsg - lenMsg); if(pLstn->bParseHost) { pMsg->msgFlags = pLstn->flags | PARSE_HOSTNAME; -- cgit v1.2.3 From 87bffff894a6ed82f126e6af33beaf53efb6b4c5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Feb 2011 16:10:21 +0100 Subject: added one more test for imuxsock to autmatted test suite control character escaping is now also being tested --- tests/Makefile.am | 10 +++++++--- tests/imuxsock_ccmiddle_root.sh | 18 ++++++++++++++++++ tests/imuxsock_traillf_root.sh | 2 +- tests/resultdata/imuxsock_ccmiddle.log | 1 + tests/syslog_inject.c | 28 ++++++++++++++++++++++++++++ tests/syslog_lf.c | 9 --------- tests/testsuites/imuxsock_ccmiddle_root.conf | 7 +++++++ 7 files changed, 62 insertions(+), 13 deletions(-) create mode 100755 tests/imuxsock_ccmiddle_root.sh create mode 100644 tests/resultdata/imuxsock_ccmiddle.log create mode 100644 tests/syslog_inject.c delete mode 100644 tests/syslog_lf.c create mode 100644 tests/testsuites/imuxsock_ccmiddle_root.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index 0ea66b0f..f66270ab 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller syslog_lf +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller syslog_inject TESTS = $(TESTRUNS) cfg.sh \ arrayqueue.sh \ linkedlistqueue.sh \ @@ -48,6 +48,7 @@ TESTS = $(TESTRUNS) cfg.sh \ dircreate_off.sh \ imuxsock_logger_root.sh \ imuxsock_traillf_root.sh \ + imuxsock_ccmiddle_root.sh \ queue-persist.sh if ENABLE_IMPTCP @@ -324,6 +325,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ imuxsock_traillf_root.sh \ testsuites/imuxsock_traillf_root.conf \ resultdata/imuxsock_traillf.log \ + imuxsock_ccmiddle_root.sh \ + testsuites/imuxsock_ccmiddle_root.conf \ + resultdata/imuxsock_ccmiddle.log \ cfg.sh ourtail_SOURCES = ourtail.c @@ -339,8 +343,8 @@ tcpflood_LDADD = $(SOL_LIBS) syslog_caller_SOURCES = syslog_caller.c syslog_caller_LDADD = $(SOL_LIBS) -syslog_lf_SOURCES = syslog_lf.c -syslog_lf_LDADD = $(SOL_LIBS) +syslog_inject_SOURCES = syslog_inject.c +syslog_inject_LDADD = $(SOL_LIBS) diagtalker_SOURCES = diagtalker.c diagtalker_LDADD = $(SOL_LIBS) diff --git a/tests/imuxsock_ccmiddle_root.sh b/tests/imuxsock_ccmiddle_root.sh new file mode 100755 index 00000000..07d23c21 --- /dev/null +++ b/tests/imuxsock_ccmiddle_root.sh @@ -0,0 +1,18 @@ +# note: we must be root and no other syslogd running in order to +# carry out this test +echo \[imuxsock_ccmiddle_root.sh\]: test trailing LF handling in imuxsock +echo This test must be run as root with no other active syslogd +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imuxsock_ccmiddle_root.conf +# send a message with trailing LF +$srcdir/syslog_inject -c +# the sleep below is needed to prevent too-early termination of rsyslogd +$srcdir/msleep 100 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! +cmp rsyslog.out.log $srcdir/resultdata/imuxsock_ccmiddle.log +if [ ! $? -eq 0 ]; then +echo "imuxsock_ccmiddle_root.sh failed" +exit 1 +fi; +source $srcdir/diag.sh exit diff --git a/tests/imuxsock_traillf_root.sh b/tests/imuxsock_traillf_root.sh index c838e0bd..02fb6d2c 100755 --- a/tests/imuxsock_traillf_root.sh +++ b/tests/imuxsock_traillf_root.sh @@ -5,7 +5,7 @@ echo This test must be run as root with no other active syslogd source $srcdir/diag.sh init source $srcdir/diag.sh startup imuxsock_traillf_root.conf # send a message with trailing LF -$srcdir/syslog_lf +$srcdir/syslog_inject -l # the sleep below is needed to prevent too-early termination of rsyslogd $srcdir/msleep 100 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages diff --git a/tests/resultdata/imuxsock_ccmiddle.log b/tests/resultdata/imuxsock_ccmiddle.log new file mode 100644 index 00000000..d2531f9d --- /dev/null +++ b/tests/resultdata/imuxsock_ccmiddle.log @@ -0,0 +1 @@ + test 1#0112 diff --git a/tests/syslog_inject.c b/tests/syslog_inject.c new file mode 100644 index 00000000..a5d77b1a --- /dev/null +++ b/tests/syslog_inject.c @@ -0,0 +1,28 @@ +/* This tool deliberately logs a message with the a trailing LF */ +/* Options: + * -l: inject linefeed at end of message + * -c: inject control character in middle of message + */ +#include +#include +#include +#include + +static inline void usage(void) { + fprintf(stderr, "Usage: syslog_inject [-l] [-c]\n"); + exit(1); +} + +int main(int argc, char *argv[]) +{ + if(argc != 2) + usage(); + if(!strcmp(argv[1], "-l")) + syslog(LOG_NOTICE, "test\n"); + else if(!strcmp(argv[1], "-c")) + syslog(LOG_NOTICE, "test 1\t2"); + else + usage(); + + return 0; +} diff --git a/tests/syslog_lf.c b/tests/syslog_lf.c deleted file mode 100644 index a140244e..00000000 --- a/tests/syslog_lf.c +++ /dev/null @@ -1,9 +0,0 @@ -/* This tool deliberately logs a message with the a trailing LF */ -#include -#include - -int main() -{ - syslog(LOG_NOTICE, "test\n"); - return 0; -} diff --git a/tests/testsuites/imuxsock_ccmiddle_root.conf b/tests/testsuites/imuxsock_ccmiddle_root.conf new file mode 100644 index 00000000..8336ecfa --- /dev/null +++ b/tests/testsuites/imuxsock_ccmiddle_root.conf @@ -0,0 +1,7 @@ +# rgerhards, 2011-02-21 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imuxsock/.libs/imuxsock + +$template outfmt,"%msg:%\n" +*.notice ./rsyslog.out.log;outfmt -- cgit v1.2.3 From de898fd5893b1b4811cc513a52af5dabcedc5291 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Feb 2011 17:33:40 +0100 Subject: bugfix (minor): warning message suggested invalid compatibility mode --- tools/syslogd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index c4a3b270..574c2efa 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2875,7 +2875,7 @@ int realMain(int argc, char **argv) if(iCompatibilityMode < 4) { errmsg.LogError(0, NO_ERRCODE, "WARNING: rsyslogd is running in compatibility mode. Automatically " "generated config directives may interfer with your rsyslog.conf settings. " - "We suggest upgrading your config and adding -c4 as the first " + "We suggest upgrading your config and adding -c5 as the first " "rsyslogd option."); } -- cgit v1.2.3 From 095a7ec73883ff14ff428653a5fa153892c1497a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 23 Feb 2011 08:45:34 +0100 Subject: preparing for 5.7.5 release --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index f771ad54..09ccf46b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.5 [V5-BETA] (rgerhards), 2011-02-?? +Version 5.7.5 [V5-BETA] (rgerhards), 2011-02-23 - enhance: imfile did not yet support multiple rulesets, now added we do this directly in the beta because a) it does not affect existing functionality and b) one may argue that this missing functionality is diff --git a/configure.ac b/configure.ac index 987711e0..aac884e2 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.4],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.5],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 4be37fd5..c8988b1a 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.4 (beta branch) of rsyslog. +

    This documentation is for version 5.7.5 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From e867cb41372e9b8e08b8eec0d663a1f3ccea5367 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 23 Feb 2011 08:54:31 +0100 Subject: added debug support for trying to find well-hidden bug --- runtime/msg.c | 10 ++++++++++ runtime/msg.h | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/runtime/msg.c b/runtime/msg.c index e8be79db..fb4d5742 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -677,6 +677,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) /* initialize members in ORDER they appear in structure (think "cache line"!) */ pM->flowCtlType = 0; pM->bDoLock = 0; + pM->bAlreadyFreed = 0; pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; @@ -803,6 +804,15 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ + /* The if below is included to try to nail down a well-hidden bug causing + * segfaults. I hope that do to the test code the problem is sooner detected and + * thus we get better data for debugging and resolving it. -- rgerhards, 2011-02-23. + * TODO: remove when no longer needed. + */ + if(pThis->bAlreadyFreed) + abort(); + pThis->bAlreadyFreed = 1; + /* end debug code */ if(pThis->pszRawMsg != pThis->szRawMsg) free(pThis->pszRawMsg); freeTAG(pThis); diff --git a/runtime/msg.h b/runtime/msg.h index 4897959c..26a07aca 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -61,7 +61,8 @@ struct msg { once data has entered the queue, this property is no longer needed. */ pthread_mutex_t mut; int iRefCount; /* reference counter (0 = unused) */ - sbool bDoLock; /* use the mutex? */ + sbool bDoLock; /* use the mutex? */ + sbool bAlreadyFreed; /* aid to help detect a well-hidden bad bug -- TODO: remove when no longer needed */ short iSeverity; /* the severity 0..7 */ short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ -- cgit v1.2.3 From ee065f1cb55be56da6ed12b35cd0e686abcb3a10 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 23 Feb 2011 09:01:57 +0100 Subject: fixed testsuite problems during make distcheck --- tests/imuxsock_ccmiddle_root.sh | 4 ++-- tests/imuxsock_logger_root.sh | 2 +- tests/imuxsock_traillf_root.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/imuxsock_ccmiddle_root.sh b/tests/imuxsock_ccmiddle_root.sh index 07d23c21..b487611a 100755 --- a/tests/imuxsock_ccmiddle_root.sh +++ b/tests/imuxsock_ccmiddle_root.sh @@ -5,9 +5,9 @@ echo This test must be run as root with no other active syslogd source $srcdir/diag.sh init source $srcdir/diag.sh startup imuxsock_ccmiddle_root.conf # send a message with trailing LF -$srcdir/syslog_inject -c +./syslog_inject -c # the sleep below is needed to prevent too-early termination of rsyslogd -$srcdir/msleep 100 +./msleep 100 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! cmp rsyslog.out.log $srcdir/resultdata/imuxsock_ccmiddle.log diff --git a/tests/imuxsock_logger_root.sh b/tests/imuxsock_logger_root.sh index e94a83fe..377999f7 100755 --- a/tests/imuxsock_logger_root.sh +++ b/tests/imuxsock_logger_root.sh @@ -7,7 +7,7 @@ source $srcdir/diag.sh startup imuxsock_logger_root.conf # send a message with trailing LF logger test # the sleep below is needed to prevent too-early termination of rsyslogd -$srcdir/msleep 100 +./msleep 100 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! cmp rsyslog.out.log $srcdir/resultdata/imuxsock_logger.log diff --git a/tests/imuxsock_traillf_root.sh b/tests/imuxsock_traillf_root.sh index 02fb6d2c..1b821ee7 100755 --- a/tests/imuxsock_traillf_root.sh +++ b/tests/imuxsock_traillf_root.sh @@ -5,9 +5,9 @@ echo This test must be run as root with no other active syslogd source $srcdir/diag.sh init source $srcdir/diag.sh startup imuxsock_traillf_root.conf # send a message with trailing LF -$srcdir/syslog_inject -l +./syslog_inject -l # the sleep below is needed to prevent too-early termination of rsyslogd -$srcdir/msleep 100 +./msleep 100 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! cmp rsyslog.out.log $srcdir/resultdata/imuxsock_traillf.log -- cgit v1.2.3 From d1eb6e0edc51a78f3209448e800b25eda50340f2 Mon Sep 17 00:00:00 2001 From: Bojan Smojver Date: Wed, 23 Feb 2011 11:25:43 +0100 Subject: added work-around for bug in gtls, which causes fd leak when using TLS The capability has been added for module to specify that they do not like being unloaded. related bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=222 Signed-off-by: Rainer Gerhards --- gss-misc.c | 1 + plugins/im3195/im3195.c | 1 + plugins/imdiag/imdiag.c | 1 + plugins/imfile/imfile.c | 1 + plugins/imgssapi/imgssapi.c | 1 + plugins/imklog/imklog.c | 1 + plugins/immark/immark.c | 1 + plugins/imptcp/imptcp.c | 1 + plugins/imrelp/imrelp.c | 1 + plugins/imsolaris/imsolaris.c | 1 + plugins/imtcp/imtcp.c | 1 + plugins/imtemplate/imtemplate.c | 1 + plugins/imudp/imudp.c | 1 + plugins/imuxsock/imuxsock.c | 1 + plugins/omdbalerting/omdbalerting.c | 1 + plugins/omgssapi/omgssapi.c | 1 + plugins/omlibdbi/omlibdbi.c | 1 + plugins/ommail/ommail.c | 1 + plugins/ommysql/ommysql.c | 1 + plugins/omoracle/omoracle.c | 1 + plugins/ompgsql/ompgsql.c | 1 + plugins/omprog/omprog.c | 1 + plugins/omrelp/omrelp.c | 1 + plugins/omruleset/omruleset.c | 1 + plugins/omsnmp/omsnmp.c | 1 + plugins/omstdout/omstdout.c | 1 + plugins/omtemplate/omtemplate.c | 1 + plugins/omtesting/omtesting.c | 1 + plugins/omudpspoof/omudpspoof.c | 1 + plugins/omuxsock/omuxsock.c | 1 + plugins/pmlastmsg/pmlastmsg.c | 1 + plugins/pmrfc3164sd/pmrfc3164sd.c | 1 + runtime/module-template.h | 12 ++++++++++ runtime/modules.c | 48 +++++++++++++++++++++++++++++++++---- runtime/modules.h | 14 +++++++++++ runtime/net.c | 1 + runtime/netstrms.c | 1 + runtime/nsd_gtls.c | 1 + runtime/nsd_ptcp.c | 1 + runtime/regexp.c | 1 + runtime/strmsrv.c | 1 + runtime/zlibw.c | 1 + tcpclt.c | 1 + tcpsrv.c | 1 + tests/rt-init.c | 1 - tools/omdiscard.c | 1 + tools/omfile.c | 1 + tools/omfwd.c | 1 + tools/ompipe.c | 1 + tools/omshell.c | 1 + tools/omusrmsg.c | 1 + tools/pmrfc3164.c | 1 + tools/pmrfc5424.c | 1 + tools/smfile.c | 1 + tools/smfwd.c | 1 + tools/smtradfile.c | 1 + tools/smtradfwd.c | 1 + 57 files changed, 123 insertions(+), 5 deletions(-) diff --git a/gss-misc.c b/gss-misc.c index a5e161de..d30eda02 100644 --- a/gss-misc.c +++ b/gss-misc.c @@ -56,6 +56,7 @@ #include "unlimited_select.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/plugins/im3195/im3195.c b/plugins/im3195/im3195.c index 106da2c8..156524c9 100644 --- a/plugins/im3195/im3195.c +++ b/plugins/im3195/im3195.c @@ -51,6 +51,7 @@ #include "errmsg.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* Module static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 81b357ef..b7a2a070 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -57,6 +57,7 @@ #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 1dd5e65c..acb58dad 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -51,6 +51,7 @@ #include "ruleset.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ +MODULE_TYPE_NOKEEP /* defines */ diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index dd3d67e3..446795d6 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -62,6 +62,7 @@ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define ALLOWEDMETHOD_GSS 2 diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index c59ce04f..69c8cd1a 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -58,6 +58,7 @@ #include "unicode-helper.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* Module static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/immark/immark.c b/plugins/immark/immark.c index 5d48369e..6410003d 100644 --- a/plugins/immark/immark.c +++ b/plugins/immark/immark.c @@ -46,6 +46,7 @@ #include "glbl.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define DEFAULT_MARK_PERIOD (20 * 60) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 6449ad62..3197564e 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -73,6 +73,7 @@ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 9be38f8f..13fd4428 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -47,6 +47,7 @@ #include "prop.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imsolaris/imsolaris.c b/plugins/imsolaris/imsolaris.c index f801833b..ee9ec5c6 100644 --- a/plugins/imsolaris/imsolaris.c +++ b/plugins/imsolaris/imsolaris.c @@ -85,6 +85,7 @@ #include "sun_cddl.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define PATH_LOG "/dev/log" diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 0cfae057..d3e9cabe 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -65,6 +65,7 @@ #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imtemplate/imtemplate.c b/plugins/imtemplate/imtemplate.c index e5e43025..0e2cac11 100644 --- a/plugins/imtemplate/imtemplate.c +++ b/plugins/imtemplate/imtemplate.c @@ -80,6 +80,7 @@ #include "debug.h" /* some debug helper functions */ MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ +MODULE_TYPE_NOKEEP /* defines */ diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index ad39ead0..56cdab28 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -54,6 +54,7 @@ #include "unicode-helper.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index ff38852c..40f32e4a 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -56,6 +56,7 @@ #include "hashtable.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define MAXFUNIX 50 diff --git a/plugins/omdbalerting/omdbalerting.c b/plugins/omdbalerting/omdbalerting.c index 2e04391c..35de5818 100644 --- a/plugins/omdbalerting/omdbalerting.c +++ b/plugins/omdbalerting/omdbalerting.c @@ -44,6 +44,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 605e5ed9..e4fdf0c0 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -58,6 +58,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 6f130f54..4b190ce3 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -50,6 +50,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index 324e1a77..886513c0 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -54,6 +54,7 @@ #include "glbl.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index aff76d0a..f8bb4aa6 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -46,6 +46,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 30b5834b..a37533ee 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -82,6 +82,7 @@ #include "omoracle.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /** */ DEF_OMOD_STATIC_DATA diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index ffdcc532..ab8e4d2c 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -49,6 +49,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 2687e7a3..56192579 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -45,6 +45,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 349e45aa..cf70381f 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -46,6 +46,7 @@ #include "debug.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omruleset/omruleset.c b/plugins/omruleset/omruleset.c index 0e0fc13b..c439bd83 100644 --- a/plugins/omruleset/omruleset.c +++ b/plugins/omruleset/omruleset.c @@ -49,6 +49,7 @@ #include "dirty.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* static data */ DEFobjCurrIf(ruleset); diff --git a/plugins/omsnmp/omsnmp.c b/plugins/omsnmp/omsnmp.c index b973b09d..443cfaab 100644 --- a/plugins/omsnmp/omsnmp.c +++ b/plugins/omsnmp/omsnmp.c @@ -47,6 +47,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 929de703..dc9912e3 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -44,6 +44,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omtemplate/omtemplate.c b/plugins/omtemplate/omtemplate.c index 5577f8c6..1472ebeb 100644 --- a/plugins/omtemplate/omtemplate.c +++ b/plugins/omtemplate/omtemplate.c @@ -45,6 +45,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omtesting/omtesting.c b/plugins/omtesting/omtesting.c index c474bb41..6d178467 100644 --- a/plugins/omtesting/omtesting.c +++ b/plugins/omtesting/omtesting.c @@ -57,6 +57,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omudpspoof/omudpspoof.c b/plugins/omudpspoof/omudpspoof.c index 3ead5447..48d7a68e 100644 --- a/plugins/omudpspoof/omudpspoof.c +++ b/plugins/omudpspoof/omudpspoof.c @@ -82,6 +82,7 @@ MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omuxsock/omuxsock.c b/plugins/omuxsock/omuxsock.c index c66e63aa..0e336c51 100644 --- a/plugins/omuxsock/omuxsock.c +++ b/plugins/omuxsock/omuxsock.c @@ -52,6 +52,7 @@ #include "unicode-helper.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/pmlastmsg/pmlastmsg.c b/plugins/pmlastmsg/pmlastmsg.c index 275f1c1f..259c5d41 100644 --- a/plugins/pmlastmsg/pmlastmsg.c +++ b/plugins/pmlastmsg/pmlastmsg.c @@ -47,6 +47,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.lastline") /* internal structures diff --git a/plugins/pmrfc3164sd/pmrfc3164sd.c b/plugins/pmrfc3164sd/pmrfc3164sd.c index 5598c025..53204ece 100644 --- a/plugins/pmrfc3164sd/pmrfc3164sd.c +++ b/plugins/pmrfc3164sd/pmrfc3164sd.c @@ -45,6 +45,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("contrib.rfc3164sd") /* internal structures diff --git a/runtime/module-template.h b/runtime/module-template.h index d05ec23c..c2585e67 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -77,6 +77,16 @@ static rsRetVal modGetType(eModType_t *modType) \ DEF_LMOD_STATIC_DATA \ MODULE_TYPE(eMOD_LIB) +/* Macro to define whether the module should be kept dynamically linked. + */ +#define MODULE_KEEP_TYPE(x)\ +static rsRetVal modGetKeepType(eModKeepType_t *modKeepType) \ + { \ + *modKeepType = x; \ + return RS_RET_OK;\ + } +#define MODULE_TYPE_NOKEEP MODULE_KEEP_TYPE(eMOD_NOKEEP) +#define MODULE_TYPE_KEEP MODULE_KEEP_TYPE(eMOD_KEEP) /* macro to define a unique module id. This must be able to fit in a void*. The * module id must be unique inside a running rsyslogd application. It is used to @@ -342,6 +352,8 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = modGetID;\ } else if(!strcmp((char*) name, "getType")) {\ *pEtryPoint = modGetType;\ + } else if(!strcmp((char*) name, "getKeepType")) {\ + *pEtryPoint = modGetKeepType;\ } /* the following definition is the standard block for queryEtryPt for output diff --git a/runtime/modules.c b/runtime/modules.c index d7362753..86e7c695 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -77,6 +77,9 @@ static pthread_mutex_t mutLoadUnload; static modInfo_t *pLoadedModules = NULL; /* list of currently-loaded modules */ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ +/* already dlopen()-ed libs */ +static struct dlhandle_s *pHandles = NULL; + /* config settings */ uchar *pModDir = NULL; /* read-only after startup */ @@ -232,7 +235,9 @@ static void moduleDestruct(modInfo_t *pThis) # ifdef VALGRIND # warning "dlclose disabled for valgrind" # else - dlclose(pThis->pModHdlr); + if (pThis->eKeepType == eMOD_NOKEEP) { + dlclose(pThis->pModHdlr); + } # endif } @@ -413,6 +418,8 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ strgen_t *pStrgen; /* used for strgen modules */ rsRetVal (*GetName)(uchar**); rsRetVal (*modGetType)(eModType_t *pType); + rsRetVal (*modGetKeepType)(eModKeepType_t *pKeepType); + struct dlhandle_s *pHandle = NULL; DEFiRet; assert(modInit != NULL); @@ -433,6 +440,8 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ */ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getType", &modGetType)); CHKiRet((*modGetType)(&pNew->eType)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getKeepType", &modGetKeepType)); + CHKiRet((*modGetKeepType)(&pNew->eKeepType)); dbgprintf("module of type %d being loaded.\n", pNew->eType); /* OK, we know we can successfully work with the module. So we now fill the @@ -529,11 +538,28 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ pNew->pszName = (uchar*) strdup((char*)name); /* we do not care if strdup() fails, we can accept that */ pNew->pModHdlr = pModHdlr; /* TODO: take this from module */ - if(pModHdlr == NULL) + if(pModHdlr == NULL) { pNew->eLinkType = eMOD_LINK_STATIC; - else + } else { pNew->eLinkType = eMOD_LINK_DYNAMIC_LOADED; + /* if we need to keep the linked module, save it */ + if (pNew->eKeepType == eMOD_KEEP) { + if((pHandle = calloc(1, sizeof (*pHandle))) == NULL) { + iRet = RS_RET_OUT_OF_MEMORY; + goto finalize_it; + } + + strncpy((char *)pHandle->szName, + (char *)name, PATH_MAX - 1); + pHandle->szName[PATH_MAX - 1] = '\0'; + pHandle->pModHdlr = pModHdlr; + pHandle->next = pHandles; + + pHandles = pHandle; + } + } + /* we initialized the structure, now let's add it to the linked list of modules */ addModToList(pNew); @@ -740,6 +766,7 @@ Load(uchar *pModName) modInfo_t *pModInfo; uchar *pModDirCurr, *pModDirNext; int iLoadCnt; + struct dlhandle_s *pHandle = NULL; assert(pModName != NULL); dbgprintf("Requested to load module '%s'\n", pModName); @@ -829,7 +856,20 @@ Load(uchar *pModName) /* complete load path constructed, so ... GO! */ dbgprintf("loading module '%s'\n", szPath); - pModHdlr = dlopen((char *) szPath, RTLD_NOW); + + /* see if we have this one already */ + for (pHandle = pHandles; pHandle; pHandle = pHandle->next) { + if (!strcmp((char *)pModName, (char *)pHandle->szName)) { + pModHdlr = pHandle->pModHdlr; + break; + } + } + + /* not found, try to dynamically link it */ + if (!pModHdlr) { + pModHdlr = dlopen((char *) szPath, RTLD_NOW); + } + iLoadCnt++; } while(pModHdlr == NULL && *pModName != '/' && pModDirNext); diff --git a/runtime/modules.h b/runtime/modules.h index df1afbc3..7eff8581 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -75,12 +75,26 @@ typedef enum eModLinkType_ { eMOD_LINK_ALL /* special: all linkage types, e.g. for unload */ } eModLinkType_t; +/* remember which shared libs we dlopen()-ed */ +struct dlhandle_s { + uchar szName[PATH_MAX]; + void *pModHdlr; + struct dlhandle_s *next; +}; + +/* should this module be kept linked? */ +typedef enum eModKeepType_ { + eMOD_NOKEEP, + eMOD_KEEP +} eModKeepType_t; + struct modInfo_s { struct modInfo_s *pPrev; /* support for creating a double linked module list */ struct modInfo_s *pNext; /* support for creating a linked module list */ int iIFVers; /* Interface version of module */ eModType_t eType; /* type of this module */ eModLinkType_t eLinkType; + eModKeepType_t eKeepType; /* keep the module dynamically linked on unload */ uchar* pszName; /* printable module name, e.g. for dbgprintf */ unsigned uRefCnt; /* reference count for this module; 0 -> may be unloaded */ /* functions supported by all types of modules */ diff --git a/runtime/net.c b/runtime/net.c index 7653ea1d..789790f6 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -69,6 +69,7 @@ #endif MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/netstrms.c b/runtime/netstrms.c index e9ff2568..ea2dd9f3 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -40,6 +40,7 @@ #include "netstrms.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 0ee70e56..152dc8de 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -56,6 +56,7 @@ GCRY_THREAD_OPTION_PTHREAD_IMPL; MODULE_TYPE_LIB +MODULE_TYPE_KEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index ca00749c..c8915231 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -52,6 +52,7 @@ #include "nsd_ptcp.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/regexp.c b/runtime/regexp.c index 86b3e6c4..21079f80 100644 --- a/runtime/regexp.c +++ b/runtime/regexp.c @@ -35,6 +35,7 @@ #include "regexp.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index a122ca8a..e66ad717 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -75,6 +75,7 @@ #include "unicode-helper.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* defines */ #define STRMSESS_MAX_DEFAULT 200 /* default for nbr of strm sessions if no number is given */ diff --git a/runtime/zlibw.c b/runtime/zlibw.c index 2b386213..455c20d9 100644 --- a/runtime/zlibw.c +++ b/runtime/zlibw.c @@ -34,6 +34,7 @@ #include "zlibw.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/tcpclt.c b/tcpclt.c index d7e30e23..0db98ea9 100644 --- a/tcpclt.c +++ b/tcpclt.c @@ -46,6 +46,7 @@ #include "srUtils.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/tcpsrv.c b/tcpsrv.c index fbb9446d..e8d79142 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -71,6 +71,7 @@ #include "unicode-helper.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* defines */ #define TCPSESS_MAX_DEFAULT 200 /* default for nbr of tcp sessions if no number is given */ diff --git a/tests/rt-init.c b/tests/rt-init.c index dbe94b4a..2d43943f 100644 --- a/tests/rt-init.c +++ b/tests/rt-init.c @@ -28,7 +28,6 @@ MODULE_TYPE_TESTBENCH - BEGINInit CODESTARTInit ENDInit diff --git a/tools/omdiscard.c b/tools/omdiscard.c index 227ad8e2..dbd18092 100644 --- a/tools/omdiscard.c +++ b/tools/omdiscard.c @@ -38,6 +38,7 @@ #include "module-template.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/tools/omfile.c b/tools/omfile.c index 8c507a56..08f965b3 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -70,6 +70,7 @@ #include "atomic.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/tools/omfwd.c b/tools/omfwd.c index 487bb35a..38a4a16b 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -64,6 +64,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/tools/ompipe.c b/tools/ompipe.c index c51a5c48..58725fb9 100644 --- a/tools/ompipe.c +++ b/tools/ompipe.c @@ -57,6 +57,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/tools/omshell.c b/tools/omshell.c index f8a68527..25f9838f 100644 --- a/tools/omshell.c +++ b/tools/omshell.c @@ -46,6 +46,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c index 768baca7..2d99c1e4 100644 --- a/tools/omusrmsg.c +++ b/tools/omusrmsg.c @@ -87,6 +87,7 @@ MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/tools/pmrfc3164.c b/tools/pmrfc3164.c index 38f556a2..635ca985 100644 --- a/tools/pmrfc3164.c +++ b/tools/pmrfc3164.c @@ -45,6 +45,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.rfc3164") /* internal structures diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c index 07994ade..2bd18042 100644 --- a/tools/pmrfc5424.c +++ b/tools/pmrfc5424.c @@ -44,6 +44,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.rfc5424") /* internal structures diff --git a/tools/smfile.c b/tools/smfile.c index 5e4a775f..1e0bf091 100644 --- a/tools/smfile.c +++ b/tools/smfile.c @@ -46,6 +46,7 @@ #include "unicode-helper.h" MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP STRGEN_NAME("RSYSLOG_FileFormat") /* internal structures diff --git a/tools/smfwd.c b/tools/smfwd.c index fe33fb2c..60fe94a7 100644 --- a/tools/smfwd.c +++ b/tools/smfwd.c @@ -43,6 +43,7 @@ #include "unicode-helper.h" MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP STRGEN_NAME("RSYSLOG_ForwardFormat") /* internal structures diff --git a/tools/smtradfile.c b/tools/smtradfile.c index eff2f99a..5484f7be 100644 --- a/tools/smtradfile.c +++ b/tools/smtradfile.c @@ -43,6 +43,7 @@ #include "unicode-helper.h" MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP STRGEN_NAME("RSYSLOG_TraditionalFileFormat") /* internal structures diff --git a/tools/smtradfwd.c b/tools/smtradfwd.c index 88dc6082..37717434 100644 --- a/tools/smtradfwd.c +++ b/tools/smtradfwd.c @@ -43,6 +43,7 @@ #include "unicode-helper.h" MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP STRGEN_NAME("RSYSLOG_TraditionalForwardFormat") /* internal structures -- cgit v1.2.3 From 7bccae395b83416be6bea2de69fdfcce46995acf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 23 Feb 2011 16:22:17 +0000 Subject: fixed compile problems on Solaris --- plugins/imuxsock/imuxsock.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index ff38852c..7ee413e7 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -606,7 +606,9 @@ static rsRetVal readSocket(lstn_t *pLstn) int iMaxLine; struct msghdr msgh; struct iovec msgiov; +# if HAVE_SCM_CREDENTIALS struct cmsghdr *cm; +# endif struct ucred *cred; uchar bufRcv[4096+1]; char aux[128]; @@ -630,11 +632,13 @@ static rsRetVal readSocket(lstn_t *pLstn) memset(&msgh, 0, sizeof(msgh)); memset(&msgiov, 0, sizeof(msgiov)); +# if HAVE_SCM_CREDENTIALS if(pLstn->bUseCreds) { memset(&aux, 0, sizeof(aux)); msgh.msg_control = aux; msgh.msg_controllen = sizeof(aux); } +# endif msgiov.iov_base = pRcv; msgiov.iov_len = iMaxLine; msgh.msg_iov = &msgiov; -- cgit v1.2.3 From bc5c178cfb3f2ecd606ecb2d1327b869e58f5f57 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 24 Feb 2011 12:33:16 +0100 Subject: bugfix: testbench failed when imptcp was not selected --- tests/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index f66270ab..1f0de728 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -13,7 +13,6 @@ TESTS = $(TESTRUNS) cfg.sh \ rulesetmultiqueue.sh \ manytcp.sh \ rsf_getenv.sh \ - manyptcp.sh \ imtcp_conndrop.sh \ imtcp_addtlframedelim.sh \ sndrcv.sh \ @@ -53,6 +52,7 @@ TESTS = $(TESTRUNS) cfg.sh \ if ENABLE_IMPTCP TESTS += \ + manyptcp.sh \ imptcp_large.sh \ imptcp_addtlframedelim.sh \ imptcp_conndrop.sh -- cgit v1.2.3 From 876294adac24792ffc95f075a9b5b79e2b99a1b2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 24 Feb 2011 14:42:14 +0100 Subject: bugfix: testbench failed when omuxsock was not selected in ./configure --- tests/Makefile.am | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 1f0de728..694b28ab 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -36,7 +36,6 @@ TESTS = $(TESTRUNS) cfg.sh \ complex1.sh \ queue-persist.sh \ pipeaction.sh \ - uxsock_simple.sh \ execonlyonce.sh \ execonlywhenprevsuspended.sh \ execonlywhenprevsuspended2.sh \ @@ -58,6 +57,10 @@ TESTS += \ imptcp_conndrop.sh endif +if ENABLE_OMUXSOCK +TESTS += uxsock_simple.sh +endif + if ENABLE_OMUDPSPOOF TESTS += sndrcv_omudpspoof.sh \ sndrcv_omudpspoof_nonstdpt.sh -- cgit v1.2.3 From f46fa5ca003b55702107e170a2868ed0120246c7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 25 Feb 2011 12:37:03 +0100 Subject: cosmetic: removing commented-out testing aid --- tools/syslogd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index 574c2efa..f7d71d06 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -699,7 +699,6 @@ msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShu assert(pBatch != NULL); pBatch->pbShutdownImmediate = pbShutdownImmediate; /* TODO: move this to batch creation! */ preprocessBatch(pBatch); -//pBatch->bSingleRuleset = 0; // TODO: testing aid, remove!!!! ruleset.ProcessBatch(pBatch); //TODO: the BATCH_STATE_COMM must be set somewhere down the road, but we //do not have this yet and so we emulate -- 2010-06-10 -- cgit v1.2.3 From 1ef709cc97d54f74d3fdeb83788cc4b01f4c6a2a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 25 Feb 2011 14:14:17 +0100 Subject: bugfix: fixed a memory leak and potential abort condition this could happen if multiple rulesets were used and some output batches contained messages belonging to more than one ruleset. fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=226 fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=218 --- ChangeLog | 5 +++++ runtime/batch.h | 8 +++++++- runtime/ruleset.c | 23 ++++++++++++++--------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5596dfbe..c4cb28b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ --------------------------------------------------------------------------- Version 5.7.6 [V5-BETA] (rgerhards), 2011-02-?? +- bugfix: fixed a memory leak and potential abort condition + this could happen if multiple rulesets were used and some output batches + contained messages belonging to more than one ruleset. + fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=226 + fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=218 - bugfix: memory leak when $RepeatedMsgReduction on was used bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=225 --------------------------------------------------------------------------- diff --git a/runtime/batch.h b/runtime/batch.h index d0504f2b..944889bd 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -136,11 +136,16 @@ batchIsValidElem(batch_t *pBatch, int i) { /* copy one batch element to another. * This creates a complete duplicate in those cases where * it is needed. Use duplication only when absolutely necessary! + * Note that all working fields are reset to zeros. If that were + * not done, we would have potential problems with invalid + * or double pointer frees. * rgerhards, 2010-06-10 */ static inline void batchCopyElem(batch_obj_t *pDest, batch_obj_t *pSrc) { - memcpy(pDest, pSrc, sizeof(batch_obj_t)); + memset(pDest, 0, sizeof(batch_obj_t)); + pDest->pUsrp = pSrc->pUsrp; + pDest->state = pSrc->state; } @@ -171,6 +176,7 @@ batchFree(batch_t *pBatch) { static inline rsRetVal batchInit(batch_t *pBatch, int maxElem) { DEFiRet; + pBatch->iDoneUpTo = 0; pBatch->maxElem = maxElem; CHKmalloc(pBatch->pElem = calloc((size_t)maxElem, sizeof(batch_obj_t))); // TODO: replace calloc by inidividual writes? diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 0584e8d6..c9c64a38 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -171,35 +171,40 @@ processBatchMultiRuleset(batch_t *pBatch) int i; int iStart; /* start index of partial batch */ int iNew; /* index for new (temporary) batch */ + int bHaveUnprocessed; /* do we (still) have unprocessed entries? (loop term predicate) */ DEFiRet; - CHKiRet(batchInit(&snglRuleBatch, pBatch->nElem)); - snglRuleBatch.pbShutdownImmediate = pBatch->pbShutdownImmediate; - - while(1) { /* loop broken inside */ + do { + bHaveUnprocessed = 0; /* search for first unprocessed element */ for(iStart = 0 ; iStart < pBatch->nElem && pBatch->pElem[iStart].state == BATCH_STATE_DISC ; ++iStart) /* just search, no action */; - if(iStart == pBatch->nElem) - FINALIZE; /* everything processed */ + break; /* everything processed */ /* prepare temporary batch */ + CHKiRet(batchInit(&snglRuleBatch, pBatch->nElem)); + snglRuleBatch.pbShutdownImmediate = pBatch->pbShutdownImmediate; currRuleset = batchElemGetRuleset(pBatch, iStart); iNew = 0; for(i = iStart ; i < pBatch->nElem ; ++i) { if(batchElemGetRuleset(pBatch, i) == currRuleset) { - batchCopyElem(&(snglRuleBatch.pElem[iNew++]), &(pBatch->pElem[i])); + /* for performance reasons, we copy only those members that we actually need */ + snglRuleBatch.pElem[iNew].pUsrp = pBatch->pElem[i].pUsrp; + snglRuleBatch.pElem[iNew].state = pBatch->pElem[i].state; + ++iNew; /* We indicate the element also as done, so it will not be processed again */ pBatch->pElem[i].state = BATCH_STATE_DISC; + } else { + bHaveUnprocessed = 1; } } snglRuleBatch.nElem = iNew; /* was left just right by the for loop */ batchSetSingleRuleset(&snglRuleBatch, 1); /* process temp batch */ processBatch(&snglRuleBatch); - } - batchFree(&snglRuleBatch); + batchFree(&snglRuleBatch); + } while(bHaveUnprocessed == 1); finalize_it: RETiRet; -- cgit v1.2.3 From 5b1cd3330196c5ffa2b8695798946aa2441b7e84 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 25 Feb 2011 14:43:30 +0100 Subject: preparing for 5.7.6 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c4cb28b4..0048c280 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.6 [V5-BETA] (rgerhards), 2011-02-?? +Version 5.7.6 [V5-BETA] (rgerhards), 2011-02-25 - bugfix: fixed a memory leak and potential abort condition this could happen if multiple rulesets were used and some output batches contained messages belonging to more than one ruleset. diff --git a/configure.ac b/configure.ac index aac884e2..09884b9c 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.5],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.6],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index c8988b1a..966c172d 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.5 (beta branch) of rsyslog. +

    This documentation is for version 5.7.6 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From bd73f263b310c29d5e0b0dd541403bde44030b86 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 1 Mar 2011 14:23:21 +0100 Subject: bugfix: potential abort condition when $RepeatedMsgReduction set to on as well as potentially in a number of other places where MsgDup() was used. This only happened when the imudp input module was used and it depended on name resolution not yet had taken place. In other words, this was a strange problem that could lead to hard to diagnose instability. So if you experience instability, chances are good that this fix will help. --- ChangeLog | 9 +++++++++ runtime/msg.c | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0048c280..b95441c7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,13 @@ --------------------------------------------------------------------------- +Version 5.7.7 [V5-BETA] (rgerhards), 2011-03-?? +- bugfix: potential abort condition when $RepeatedMsgReduction set to on + as well as potentially in a number of other places where MsgDup() was + used. This only happened when the imudp input module was used and it + depended on name resolution not yet had taken place. In other words, + this was a strange problem that could lead to hard to diagnose + instability. So if you experience instability, chances are good that + this fix will help. +--------------------------------------------------------------------------- Version 5.7.6 [V5-BETA] (rgerhards), 2011-02-25 - bugfix: fixed a memory leak and potential abort condition this could happen if multiple rulesets were used and some output batches diff --git a/runtime/msg.c b/runtime/msg.c index fb4d5742..b0261faa 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -932,13 +932,14 @@ msg_t* MsgDup(msg_t* pOld) pNew->iLenMSG = pOld->iLenMSG; pNew->iLenTAG = pOld->iLenTAG; pNew->iLenHOSTNAME = pOld->iLenHOSTNAME; - if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) { + if((pOld->msgFlags & NEEDS_DNSRESOL)) { localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet); if(localRet != RS_RET_OK) { /* if something fails, we accept loss of this property, it is * better than losing the whole message. */ pNew->msgFlags &= ~NEEDS_DNSRESOL; + pNew->rcvFrom.pRcvFrom = NULL; /* make sure no dangling values */ } } else { if(pOld->rcvFrom.pRcvFrom != NULL) { -- cgit v1.2.3 From 3018b443e7aee3de5485fa00e01951d2a05f2774 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 2 Mar 2011 14:01:48 +0100 Subject: preparing for 5.7.7 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index b95441c7..4344535f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.7 [V5-BETA] (rgerhards), 2011-03-?? +Version 5.7.7 [V5-BETA] (rgerhards), 2011-03-02 - bugfix: potential abort condition when $RepeatedMsgReduction set to on as well as potentially in a number of other places where MsgDup() was used. This only happened when the imudp input module was used and it diff --git a/configure.ac b/configure.ac index 09884b9c..278288e7 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.7],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 966c172d..48ff5e16 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.6 (beta branch) of rsyslog. +

    This documentation is for version 5.7.7 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 87f8ac6dc4343eb4664ca5e234c163c399afca1b Mon Sep 17 00:00:00 2001 From: Bojan Smojver Date: Fri, 4 Mar 2011 07:55:53 +0100 Subject: bugfix: regression: memory leak in module loader Signed-off-by: Rainer Gerhards --- runtime/modules.c | 28 ++++++++++++++++++---------- runtime/modules.h | 6 +++--- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/runtime/modules.c b/runtime/modules.c index 86e7c695..4541bddf 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -545,18 +545,26 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ /* if we need to keep the linked module, save it */ if (pNew->eKeepType == eMOD_KEEP) { - if((pHandle = calloc(1, sizeof (*pHandle))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto finalize_it; + /* see if we have this one already */ + for (pHandle = pHandles; pHandle; pHandle = pHandle->next) { + if (!strcmp((char *)name, (char *)pHandle->pszName)) + break; } - strncpy((char *)pHandle->szName, - (char *)name, PATH_MAX - 1); - pHandle->szName[PATH_MAX - 1] = '\0'; - pHandle->pModHdlr = pModHdlr; - pHandle->next = pHandles; + /* not found, create it */ + if (!pHandle) { + if((pHandle = malloc(sizeof (*pHandle))) == NULL) { + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + if((pHandle->pszName = (uchar*) strdup((char*)name)) == NULL) { + free(pHandle); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + pHandle->pModHdlr = pModHdlr; + pHandle->next = pHandles; - pHandles = pHandle; + pHandles = pHandle; + } } } @@ -859,7 +867,7 @@ Load(uchar *pModName) /* see if we have this one already */ for (pHandle = pHandles; pHandle; pHandle = pHandle->next) { - if (!strcmp((char *)pModName, (char *)pHandle->szName)) { + if (!strcmp((char *)pModName, (char *)pHandle->pszName)) { pModHdlr = pHandle->pModHdlr; break; } diff --git a/runtime/modules.h b/runtime/modules.h index 7eff8581..4daaf1f9 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -77,9 +77,9 @@ typedef enum eModLinkType_ { /* remember which shared libs we dlopen()-ed */ struct dlhandle_s { - uchar szName[PATH_MAX]; - void *pModHdlr; - struct dlhandle_s *next; + uchar *pszName; + void *pModHdlr; + struct dlhandle_s *next; }; /* should this module be kept linked? */ -- cgit v1.2.3 From b8610a107ee4fd9e4b1ba06e7bd43dd6cbd72ea7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 4 Mar 2011 09:19:08 +0100 Subject: added first experimental skeleton of what is to become a testconfig generator currently it generates only the power set, more work to do to make it a real config generator. --- tests/testconfgen.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/testconfgen.c diff --git a/tests/testconfgen.c b/tests/testconfgen.c new file mode 100644 index 00000000..9f191cc7 --- /dev/null +++ b/tests/testconfgen.c @@ -0,0 +1,72 @@ +/* a testcase generator + * THis program reads stdin, which must consist of (name,stmt) tupels + * where name is a part of the config name (small!) and stmt is an actual + * config statement. These tupels must be encoded as + * namestmt + * on stdin. After all tupels are read, the power set of all possible + * configurations is generated. + * Copyright (C) 2011 by Rainer Gerhards and Adiscon GmbH + * Released under the GPLv3 as part of the rsyslog project. + */ +#include +#include +#include + +static int arr[128]; +static char *name[128]; +static char *stmt[128]; + +void output(int n) +{ + int i; + + printf("name:"); + for(i = 0 ; i < n ; ++i) { + if(arr[i]) { + printf("-%s", name[i]); + } + } + printf("\n"); +} + +void pows(int n, int i) +{ + if(i == 0) { + output(n); + } else { + --i; + arr[i] = 0; + pows(n, i); + arr[i] = 1; + pows(n, i); + } +} + + +int main(int argc, char *argv[]) +{ + int n; + char iname[512]; + char istmt[2048]; + int nscanned; + + n = 0; + while(!feof(stdin)) { + nscanned = scanf("%s %[^\n]s\n", iname, istmt); + if(nscanned == EOF) + break; + else if(nscanned != 2) { + fprintf(stderr, "problem scanning entry %d, scanned %d\n", + n, nscanned); + exit(1); + } + name[n] = strdup(iname); + stmt[n] = strdup(istmt); + n++; + printf("name: %s, stmt: %s\n", iname, istmt); + } + /* n is on to high for an index, but just right as the actual number! */ + + printf("read %d entries\n", n); + pows(n, n); +} -- cgit v1.2.3 From eb8e018d5a7b27700bbae1b8681b4f2f8e8a4f99 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 6 Mar 2011 15:05:12 +0100 Subject: updated ChangLog with merged in fix --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4344535f..52f0528d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,8 @@ --------------------------------------------------------------------------- +Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-?? +- bugfix: file descriptor leak in gnutls netstream driver + fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=222 +--------------------------------------------------------------------------- Version 5.7.7 [V5-BETA] (rgerhards), 2011-03-02 - bugfix: potential abort condition when $RepeatedMsgReduction set to on as well as potentially in a number of other places where MsgDup() was -- cgit v1.2.3 From 68124ef1f63daef07ecf45d479ca9697cb099441 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 6 Mar 2011 15:18:01 +0100 Subject: fixed small regression from gnutls fd leak fix --- plugins/impstats/impstats.c | 1 + plugins/pmcisconames/pmcisconames.c | 1 + plugins/pmsnare/pmsnare.c | 1 + 3 files changed, 3 insertions(+) diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index 1312a4e8..aa98ae9d 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -49,6 +49,7 @@ #include "prop.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define DEFAULT_STATS_PERIOD (5 * 60) diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c index 4171e688..61688cbf 100644 --- a/plugins/pmcisconames/pmcisconames.c +++ b/plugins/pmcisconames/pmcisconames.c @@ -41,6 +41,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.cisconames") /* internal structures diff --git a/plugins/pmsnare/pmsnare.c b/plugins/pmsnare/pmsnare.c index 4a9880d4..f3658d11 100644 --- a/plugins/pmsnare/pmsnare.c +++ b/plugins/pmsnare/pmsnare.c @@ -58,6 +58,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.snare") /* internal structures -- cgit v1.2.3 From 4fb7dd296efc6a798e884f0504fdf0bedebba9f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 6 Mar 2011 15:50:48 +0100 Subject: backported new testbench support for valgrind --- tests/diag.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/diag.sh b/tests/diag.sh index 2f307750..64b507e6 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -34,7 +34,12 @@ case $1 in ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use # returns only after successful startup, $3 is the instance (blank or 2!) - $valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $valgrind ../tools/rsyslogd -c6 -u2 -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $srcdir/diag.sh wait-startup $3 + ;; + 'startup-vg') # start rsyslogd with default params under valgrind control. $2 is the config file name to use + # returns only after successful startup, $3 is the instance (blank or 2!) + valgrind --error-exitcode=10 --malloc-fill=ff --free-fill=fe --leak-check=full ../tools/rsyslogd -c6 -u2 -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & $srcdir/diag.sh wait-startup $3 ;; 'wait-startup') # wait for rsyslogd startup ($2 is the instance) @@ -58,6 +63,18 @@ case $1 in exit 1 fi ;; + 'wait-shutdown-vg') # actually, we wait for rsyslog.pid to be deleted. $2 is the + # instance + wait `cat rsyslog.pid` + export RSYSLOGD_EXIT=$? + echo rsyslogd run exited with $RSYSLOGD_EXIT + if [ -e core.* ] + then + echo "ABORT! core file exists, starting interactive shell" + bash + exit 1 + fi + ;; 'wait-queueempty') # wait for main message queue to be empty. $2 is the instance. if [ "$2" == "2" ] then -- cgit v1.2.3 From 4d55e2481f51b59fff9a32f9392bb3e74da9789a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 6 Mar 2011 15:59:06 +0100 Subject: forgot to backport imdiag mem leak fix -- now done --- plugins/imdiag/imdiag.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index b7a2a070..0a69ee43 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -292,6 +292,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) { int iMsgQueueSize; uchar *pszMsg; + uchar *pToFree = NULL; uchar cmdBuf[1024]; DEFiRet; @@ -303,6 +304,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) * before proceeding. */ CHKmalloc(pszMsg = MALLOC(sizeof(uchar) * (iLenMsg + 1))); + pToFree = pszMsg; memcpy(pszMsg, pRcv, iLenMsg); pszMsg[iLenMsg] = '\0'; @@ -322,6 +324,8 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) } finalize_it: + if(pToFree != NULL) + free(pToFree); RETiRet; } -- cgit v1.2.3 From 36899ac8c40e78767ba58e440167a76dd4a73db1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 6 Mar 2011 16:09:17 +0100 Subject: added new test to check RepeatedMsgReduction (via TCP) This is kind of a backport of a similar UDP-based test from v6. However, we do not have everything required for UDP in the v5 test tool base, so we mimic the test with tcp -- which is not bad at all, because the code path for TCP is somewhat different (and thus once we merge this patch into v6, it is a useful addition). --- tests/Makefile.am | 3 +++ tests/tcp-msgreduc-vg.sh | 20 ++++++++++++++++++++ tests/testsuites/tcp-msgreduc-vg.conf | 10 ++++++++++ 3 files changed, 33 insertions(+) create mode 100755 tests/tcp-msgreduc-vg.sh create mode 100644 tests/testsuites/tcp-msgreduc-vg.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index 694b28ab..2e2e47aa 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -13,6 +13,7 @@ TESTS = $(TESTRUNS) cfg.sh \ rulesetmultiqueue.sh \ manytcp.sh \ rsf_getenv.sh \ + tcp-msgreduc-vg.sh \ imtcp_conndrop.sh \ imtcp_addtlframedelim.sh \ sndrcv.sh \ @@ -210,6 +211,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/imtcp_conndrop.conf \ imtcp_addtlframedelim.sh \ testsuites/imtcp_addtlframedelim.conf \ + tcp-msgreduc-vg.sh \ + testsuites/./tcp-msgreduc-vg.conf \ inputname.sh \ testsuites/inputname_imtcp.conf \ testsuites/1.inputname_imtcp_12514 \ diff --git a/tests/tcp-msgreduc-vg.sh b/tests/tcp-msgreduc-vg.sh new file mode 100755 index 00000000..82747f3b --- /dev/null +++ b/tests/tcp-msgreduc-vg.sh @@ -0,0 +1,20 @@ +# check if valgrind violations occur. Correct output is not checked. +# added 2011-03-01 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[tcp-msgreduc-vg.sh\]: testing msg reduction via UDP +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg tcp-msgreduc-vg.conf +source $srcdir/diag.sh wait-startup +./tcpflood -t 127.0.0.1 -m 4 -r -M "<133>2011-03-01T11:22:12Z host tag msgh ..." +./tcpflood -t 127.0.0.1 -m 1 -r -M "<133>2011-03-01T11:22:12Z host tag msgh ...x" +# we need to give rsyslog a little time to settle the receiver +./msleep 1500 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown-vg +if [ "$RSYSLOGD_EXIT" -eq "10" ] +then + echo "tcp-msgreduc-vg.sh FAILED" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/testsuites/tcp-msgreduc-vg.conf b/tests/testsuites/tcp-msgreduc-vg.conf new file mode 100644 index 00000000..72420f04 --- /dev/null +++ b/tests/testsuites/tcp-msgreduc-vg.conf @@ -0,0 +1,10 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-05-22 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 +$RepeatedMsgReduction on + +$template outfmt,"%msg:F,58:2%\n" +*.* ./rsyslog.out.log;outfmt -- cgit v1.2.3 From f813c01258ec5ef3adc8f4790c5c743c2f4b462a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 7 Mar 2011 18:55:13 +0100 Subject: bugfix: discard action did not work under some circumstances fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=217 --- ChangeLog | 2 ++ action.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 52f0528d..5f9b06c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-?? +- bugfix: discard action did not work under some circumstances + fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=217 - bugfix: file descriptor leak in gnutls netstream driver fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=222 --------------------------------------------------------------------------- diff --git a/action.c b/action.c index 36830577..c9e5e095 100644 --- a/action.c +++ b/action.c @@ -1481,7 +1481,8 @@ helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch) DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod)); for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - if(pBatch->pElem[i].bFilterOK) { + if( pBatch->pElem[i].bFilterOK + && pBatch->pElem[i].state != BATCH_STATE_DISC) { doActionCallAction(pAction, (msg_t*)(pBatch->pElem[i].pUsrp)); } } -- cgit v1.2.3 From 1a0875865300472150c411afa8335e47a1722bae Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 8 Mar 2011 18:58:28 +0100 Subject: further improved testbench some cosmetic issues, plus a new valgrind-based test --- tests/Makefile.am | 2 ++ tests/diag.sh | 7 +++++++ tests/discard-rptdmsg-vg.sh | 13 +++++++++++++ tests/discard-rptdmsg.sh | 7 ------- tests/tcp-msgreduc-vg.sh | 6 +----- 5 files changed, 23 insertions(+), 12 deletions(-) create mode 100755 tests/discard-rptdmsg-vg.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index c834ef6e..3e07e1c6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -49,6 +49,7 @@ TESTS = $(TESTRUNS) cfg.sh \ imuxsock_traillf_root.sh \ imuxsock_ccmiddle_root.sh \ discard-rptdmsg.sh \ + discard-rptdmsg-vg.sh \ discard.sh \ queue-persist.sh @@ -223,6 +224,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ discard.sh \ testsuites/discard.conf \ discard-rptdmsg.sh \ + discard-rptdmsg-vg.sh \ testsuites/discard-rptdmsg.conf \ diag.sh \ testsuites/diag-common.conf \ diff --git a/tests/diag.sh b/tests/diag.sh index 64b507e6..e8e3ce1c 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -75,6 +75,13 @@ case $1 in exit 1 fi ;; + 'check-exit-vg') # wait for main message queue to be empty. $2 is the instance. + if [ "$RSYSLOGD_EXIT" -eq "10" ] + then + echo "valgrind run FAILED with exceptions - terminating" + exit 1 + fi + ;; 'wait-queueempty') # wait for main message queue to be empty. $2 is the instance. if [ "$2" == "2" ] then diff --git a/tests/discard-rptdmsg-vg.sh b/tests/discard-rptdmsg-vg.sh new file mode 100755 index 00000000..f56ac597 --- /dev/null +++ b/tests/discard-rptdmsg-vg.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[discard-rptdmsg.sh\]: testing discard-rptdmsg functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg discard-rptdmsg.conf +source $srcdir/diag.sh tcpflood -m10 -i1 +# we need to give rsyslog a little time to settle the receiver +./msleep 1500 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +source $srcdir/diag.sh seq-check 2 10 +source $srcdir/diag.sh exit diff --git a/tests/discard-rptdmsg.sh b/tests/discard-rptdmsg.sh index a8b35c38..a8be110c 100755 --- a/tests/discard-rptdmsg.sh +++ b/tests/discard-rptdmsg.sh @@ -1,15 +1,8 @@ -# Test for discard-rptdmsg functionality -# This test checks if discard-rptdmsg works. It is not a perfect test but -# will find at least segfaults and obviously not discard-rptdmsged messages. -# added 2009-07-30 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 -# uncomment for debugging support: echo =============================================================================== echo \[discard-rptdmsg.sh\]: testing discard-rptdmsg functionality source $srcdir/diag.sh init source $srcdir/diag.sh startup discard-rptdmsg.conf -# 20000 messages should be enough - the disk test is slow enough ;) -sleep 4 source $srcdir/diag.sh tcpflood -m10 -i1 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown diff --git a/tests/tcp-msgreduc-vg.sh b/tests/tcp-msgreduc-vg.sh index 82747f3b..7e388360 100755 --- a/tests/tcp-msgreduc-vg.sh +++ b/tests/tcp-msgreduc-vg.sh @@ -12,9 +12,5 @@ source $srcdir/diag.sh wait-startup ./msleep 1500 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown-vg -if [ "$RSYSLOGD_EXIT" -eq "10" ] -then - echo "tcp-msgreduc-vg.sh FAILED" - exit 1 -fi +source $srcdir/diag.sh wait-shutdown-vg source $srcdir/diag.sh exit -- cgit v1.2.3 From 22f36ffbbd0c4e0fe1bfe331cb7ca15ec6ae80aa Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 08:23:16 +0100 Subject: testbench: valgrind tests only executed if valgrind is available --- configure.ac | 4 ++++ tests/Makefile.am | 8 ++++++-- tests/imtcp_conndrop.sh | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 278288e7..a7d26a1f 100644 --- a/configure.ac +++ b/configure.ac @@ -13,6 +13,10 @@ AC_CONFIG_HEADERS([config.h]) AC_GNU_SOURCE +# check if valgrind is present +AC_CHECK_PROG(have_valgrind, [valgrind], [yes]) +AM_CONDITIONAL(HAVE_VALGRIND, test x$have_valgrind = xyes) + # check for Java compiler AC_CHECK_PROG(HAVE_JAVAC, [javac], [yes]) if test x"$HAVE_JAVAC" = x""; then diff --git a/tests/Makefile.am b/tests/Makefile.am index 3e07e1c6..7527b27f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -13,7 +13,6 @@ TESTS = $(TESTRUNS) cfg.sh \ rulesetmultiqueue.sh \ manytcp.sh \ rsf_getenv.sh \ - tcp-msgreduc-vg.sh \ imtcp_conndrop.sh \ imtcp_addtlframedelim.sh \ sndrcv.sh \ @@ -49,10 +48,15 @@ TESTS = $(TESTRUNS) cfg.sh \ imuxsock_traillf_root.sh \ imuxsock_ccmiddle_root.sh \ discard-rptdmsg.sh \ - discard-rptdmsg-vg.sh \ discard.sh \ queue-persist.sh +if HAVE_VALGRIND +TESTS += \ + discard-rptdmsg-vg.sh \ + tcp-msgreduc-vg.sh +endif + if ENABLE_IMPTCP TESTS += \ manyptcp.sh \ diff --git a/tests/imtcp_conndrop.sh b/tests/imtcp_conndrop.sh index 9855ee3b..c7eb89e8 100755 --- a/tests/imtcp_conndrop.sh +++ b/tests/imtcp_conndrop.sh @@ -9,7 +9,7 @@ source $srcdir/diag.sh init source $srcdir/diag.sh startup imtcp_conndrop.conf # 100 byte messages to gain more practical data use source $srcdir/diag.sh tcpflood -c20 -m50000 -r -d100 -P129 -D -sleep 4 # due to large messages, we need this time for the tcp receiver to settle... +sleep 6 # due to large messages, we need this time for the tcp receiver to settle... source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown # and wait for it to terminate source $srcdir/diag.sh seq-check 0 49999 -E -- cgit v1.2.3 From 906b04e97e8e1f8f09110e6e13caad87862f4736 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 7 Mar 2011 22:39:37 +0100 Subject: systemd: use standard syslog.socket unit In systemd we now have a standard socket unit for /dev/log called syslog.socket. This unit can be shared between an early boot mini syslog and the full syslog implementation started later on. The mini syslog is shipped along systemd and does nothing but simply forward the data received through /dev/log to the kernel log buffer (i.e. kmsg, as visible by dmesg). It is run during early boot, and then as soon as rsyslog starts up it is terminated, so that rsyslog can take over the /dev/log socket. Since one of the first things rsyslog does after starting up is flushing the kernel log buffer to disk we end up with all data from early boot up in syslog. This patch changes two things: removes rsyslog.socket and instead configures rsyslog.service to take possession of syslock.socket. And secondly includes a PreStart line to terminate the running syslog bridge instance. --- Makefile.am | 3 --- rsyslog.service.in | 3 ++- rsyslog.socket | 8 -------- 3 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 rsyslog.socket diff --git a/Makefile.am b/Makefile.am index 9493e373..f699cc46 100644 --- a/Makefile.am +++ b/Makefile.am @@ -44,9 +44,6 @@ endif # if HAVE_SYSTEMD -dist_systemdsystemunit_DATA = \ - rsyslog.socket - nodist_systemdsystemunit_DATA = \ rsyslog.service diff --git a/rsyslog.service.in b/rsyslog.service.in index 2bcde528..03db596e 100644 --- a/rsyslog.service.in +++ b/rsyslog.service.in @@ -2,9 +2,10 @@ Description=System Logging Service [Service] +ExecStartPre=/bin/systemctl stop systemd-kmsg-syslogd.service ExecStart=@sbindir@/rsyslogd -n -c5 ExecReload=/bin/kill -HUP $MAINPID +Sockets=syslog.socket [Install] WantedBy=multi-user.target -Also=rsyslog.socket diff --git a/rsyslog.socket b/rsyslog.socket deleted file mode 100644 index 0cd86054..00000000 --- a/rsyslog.socket +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Syslog Socket - -[Socket] -ListenDatagram=/dev/log - -[Install] -WantedBy=sockets.target -- cgit v1.2.3 From 3f657f5b056c02cddb17ad639b226f24ebb047c5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 09:58:07 +0100 Subject: make testbench not errout if imdiag is not enabled --- tests/Makefile.am | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 7527b27f..a720c0c3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,7 +1,10 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller syslog_inject -TESTS = $(TESTRUNS) cfg.sh \ +TESTS = $(TESTRUNS) cfg.sh + +if ENABLE_IMDIAG +TESTS += \ arrayqueue.sh \ linkedlistqueue.sh \ da-mainmsg-q.sh \ @@ -49,7 +52,10 @@ TESTS = $(TESTRUNS) cfg.sh \ imuxsock_ccmiddle_root.sh \ discard-rptdmsg.sh \ discard.sh \ - queue-persist.sh + queue-persist.sh \ + arrayqueue.sh \ + linkedlistqueue.sh +endif if HAVE_VALGRIND TESTS += \ @@ -89,9 +95,11 @@ TESTS += omod-if-array.sh \ endif if ENABLE_OMRULESET +if ENABLE_IMDIAG TESTS += omruleset.sh \ omruleset-queue.sh endif +endif if ENABLE_EXTENDED_TESTS TESTS += random.sh @@ -193,9 +201,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rsf_getenv.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ - arrayqueue.sh \ testsuites/arrayqueue.conf \ - linkedlistqueue.sh \ testsuites/linkedlistqueue.conf \ da-mainmsg-q.sh \ testsuites/da-mainmsg-q.conf \ -- cgit v1.2.3 From 67a52bbd1d236f2f24297efdbceaf20c6fa10906 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 10:01:19 +0100 Subject: preparing for 5.7.8 release --- ChangeLog | 3 ++- configure.ac | 2 +- doc/manual.html | 2 +- tests/imtcp_conndrop.sh | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index c5058fbc..baf0bcfe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- -Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-?? +Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-09 +- systemd support somewhat improved (can now take over existing log sockt) - bugfix: discard action did not work under some circumstances fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=217 - bugfix: file descriptor leak in gnutls netstream driver diff --git a/configure.ac b/configure.ac index a7d26a1f..b5666d66 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.7],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.8],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 48ff5e16..945a5a62 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.7 (beta branch) of rsyslog. +

    This documentation is for version 5.7.8 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might diff --git a/tests/imtcp_conndrop.sh b/tests/imtcp_conndrop.sh index c7eb89e8..0bfcd99c 100755 --- a/tests/imtcp_conndrop.sh +++ b/tests/imtcp_conndrop.sh @@ -9,7 +9,7 @@ source $srcdir/diag.sh init source $srcdir/diag.sh startup imtcp_conndrop.conf # 100 byte messages to gain more practical data use source $srcdir/diag.sh tcpflood -c20 -m50000 -r -d100 -P129 -D -sleep 6 # due to large messages, we need this time for the tcp receiver to settle... +sleep 10 # due to large messages, we need this time for the tcp receiver to settle... source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown # and wait for it to terminate source $srcdir/diag.sh seq-check 0 49999 -E -- cgit v1.2.3 From 53afa44de8d7796df46a12d66fd9866165779d94 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 10:31:47 +0100 Subject: bugfix: (regression) omhdfs did no longer compile --- ChangeLog | 3 +++ plugins/omhdfs/omhdfs.c | 1 + 2 files changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index baf0bcfe..a9fb0d34 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? +- bugfix: (regression) omhdfs did no longer compile +--------------------------------------------------------------------------- Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-09 - systemd support somewhat improved (can now take over existing log sockt) - bugfix: discard action did not work under some circumstances diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 9705b7fd..8b72747f 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -50,6 +50,7 @@ #include "hashtable_itr.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ -- cgit v1.2.3 From d1ae9e393885fb6d9fc69fc1bb08a098a8ac0328 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 11:28:22 +0100 Subject: improved testbench --- ChangeLog | 1 + tests/Makefile.am | 5 +++++ tests/discard-allmark-vg.sh | 13 +++++++++++++ tests/discard-allmark.sh | 10 ++++++++++ tests/testsuites/discard-allmark.conf | 15 +++++++++++++++ 5 files changed, 44 insertions(+) create mode 100755 tests/discard-allmark-vg.sh create mode 100755 tests/discard-allmark.sh create mode 100644 tests/testsuites/discard-allmark.conf diff --git a/ChangeLog b/ChangeLog index a9fb0d34..5be02fca 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? +- improved testbench - bugfix: (regression) omhdfs did no longer compile --------------------------------------------------------------------------- Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-09 diff --git a/tests/Makefile.am b/tests/Makefile.am index a720c0c3..35513614 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -51,6 +51,7 @@ TESTS += \ imuxsock_traillf_root.sh \ imuxsock_ccmiddle_root.sh \ discard-rptdmsg.sh \ + discard-allmark.sh \ discard.sh \ queue-persist.sh \ arrayqueue.sh \ @@ -60,6 +61,7 @@ endif if HAVE_VALGRIND TESTS += \ discard-rptdmsg-vg.sh \ + discard-allmark-vg.sh \ tcp-msgreduc-vg.sh endif @@ -236,6 +238,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ discard-rptdmsg.sh \ discard-rptdmsg-vg.sh \ testsuites/discard-rptdmsg.conf \ + discard-allmark.sh \ + discard-allmark-vg.sh \ + testsuites/discard-allmark.conf \ diag.sh \ testsuites/diag-common.conf \ testsuites/diag-common2.conf \ diff --git a/tests/discard-allmark-vg.sh b/tests/discard-allmark-vg.sh new file mode 100755 index 00000000..57ce8e3e --- /dev/null +++ b/tests/discard-allmark-vg.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[discard-allmark.sh\]: testing discard-allmark functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg discard-allmark.conf +source $srcdir/diag.sh tcpflood -m10 -i1 +# we need to give rsyslog a little time to settle the receiver +./msleep 1500 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +source $srcdir/diag.sh seq-check 2 10 +source $srcdir/diag.sh exit diff --git a/tests/discard-allmark.sh b/tests/discard-allmark.sh new file mode 100755 index 00000000..eb46ae70 --- /dev/null +++ b/tests/discard-allmark.sh @@ -0,0 +1,10 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[discard-allmark.sh\]: testing discard-allmark functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup discard-allmark.conf +source $srcdir/diag.sh tcpflood -m10 -i1 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 2 10 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/discard-allmark.conf b/tests/testsuites/discard-allmark.conf new file mode 100644 index 00000000..8a4983c1 --- /dev/null +++ b/tests/testsuites/discard-allmark.conf @@ -0,0 +1,15 @@ +# Test for discard functionality +# rgerhards, 2009-07-30 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ActionWriteAllMarkMessages on + +:msg, contains, "00000001" ~ + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit v1.2.3 From 533a5351365cec93f233e0e99d98cedeadc5bd69 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 12:40:58 +0100 Subject: added MySQL life tests for ommysql tests run against a life database instance which must be setup in a specific way. --- ChangeLog | 2 ++ configure.ac | 18 ++++++++++++++++++ tests/Makefile.am | 19 +++++++++++++++++++ tests/mysql-asyn-vg.sh | 14 ++++++++++++++ tests/mysql-asyn.sh | 13 +++++++++++++ tests/mysql-basic-vg.sh | 14 ++++++++++++++ tests/mysql-basic.sh | 13 +++++++++++++ tests/tcp-msgreduc-vg.sh | 2 +- tests/testsuites/mysql-asyn.conf | 5 +++++ tests/testsuites/mysql-basic.conf | 4 ++++ tests/testsuites/mysql-select-msg.sql | 2 ++ tests/testsuites/mysql-truncate.sql | 2 ++ 12 files changed, 107 insertions(+), 1 deletion(-) create mode 100755 tests/mysql-asyn-vg.sh create mode 100755 tests/mysql-asyn.sh create mode 100755 tests/mysql-basic-vg.sh create mode 100755 tests/mysql-basic.sh create mode 100644 tests/testsuites/mysql-asyn.conf create mode 100644 tests/testsuites/mysql-basic.conf create mode 100644 tests/testsuites/mysql-select-msg.sql create mode 100644 tests/testsuites/mysql-truncate.sql diff --git a/ChangeLog b/ChangeLog index 5be02fca..69620435 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,8 @@ --------------------------------------------------------------------------- Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? - improved testbench + among others, life tests for ommysql (against a test database) have + been added, valgrind-based testing enhanced, ... - bugfix: (regression) omhdfs did no longer compile --------------------------------------------------------------------------- Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-09 diff --git a/configure.ac b/configure.ac index b5666d66..58b203ff 100644 --- a/configure.ac +++ b/configure.ac @@ -736,6 +736,23 @@ AC_ARG_ENABLE(extended_tests, AM_CONDITIONAL(ENABLE_EXTENDED_TESTS, test x$enable_extended_tests = xyes) +# capability to enable MySQL testbench tests. This requries that a Syslog database +# with the default schema has been created on the local (127.0.0.1) MySQL server and +# a user "rsyslog" with password "testbench" exists, is able to login with default +# parameters and has sufficient (read: all) privileges on that database. +# rgerhards, 2011-03-09 +AC_ARG_ENABLE(mysql_tests, + [AS_HELP_STRING([--enable-mysql-tests],[enable MySQL specific tests in testbench @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_mysql_tests="yes" ;; + no) enable_mysql_tests="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-mysql-tests) ;; + esac], + [enable_mysql_tests=no] +) +AM_CONDITIONAL(ENABLE_MYSQL_TESTS, test x$enable_mysql_tests = xyes) + + # Mail support (so far we do not need a library, but we need to turn this on and off) AC_ARG_ENABLE(mail, [AS_HELP_STRING([--enable-mail],[Enable mail support @<:@default=no@:>@])], @@ -1221,6 +1238,7 @@ echo echo "---{ debugging support }---" echo " Testbench enabled: $enable_testbench" echo " Extended Testbench enabled: $enable_extended_tests" +echo " MySQL Tests enabled: $enable_mysql_tests" echo " Debug mode enabled: $enable_debug" echo " Runtime Instrumentation enabled: $enable_rtinst" echo " Diagnostic tools enabled: $enable_diagtools" diff --git a/tests/Makefile.am b/tests/Makefile.am index 35513614..acadda04 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -65,6 +65,17 @@ TESTS += \ tcp-msgreduc-vg.sh endif +if ENABLE_MYSQL_TESTS +TESTS += \ + mysql-basic.sh \ + mysql-asyn.sh +if HAVE_VALGRIND +TESTS += \ + mysql-basic-vg.sh \ + mysql-asyn-vg.sh +endif +endif + if ENABLE_IMPTCP TESTS += \ manyptcp.sh \ @@ -354,6 +365,14 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ imuxsock_ccmiddle_root.sh \ testsuites/imuxsock_ccmiddle_root.conf \ resultdata/imuxsock_ccmiddle.log \ + testsuites\mysql-truncate.sql \ + testsuites\mysql-select-msg.sql \ + mysql-basic.sh \ + mysql-basic-vg.sh \ + testsuites\mysql-basic.sh \ + mysql-asyn.sh \ + mysql-asyn-vg.sh \ + testsuites\mysql-asyn.sh \ cfg.sh ourtail_SOURCES = ourtail.c diff --git a/tests/mysql-asyn-vg.sh b/tests/mysql-asyn-vg.sh new file mode 100755 index 00000000..dfe68665 --- /dev/null +++ b/tests/mysql-asyn-vg.sh @@ -0,0 +1,14 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[mysql-asyn.sh\]: asyn test for mysql functionality +source $srcdir/diag.sh init +mysql --user=rsyslog --password=testbench < testsuites/mysql-truncate.sql +source $srcdir/diag.sh startup-vg mysql-asyn.conf +source $srcdir/diag.sh injectmsg 0 50000 +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +# note "-s" is requried to suppress the select "field header" +mysql -s --user=rsyslog --password=testbench < testsuites/mysql-select-msg.sql > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 49999 +source $srcdir/diag.sh exit diff --git a/tests/mysql-asyn.sh b/tests/mysql-asyn.sh new file mode 100755 index 00000000..de6d6fde --- /dev/null +++ b/tests/mysql-asyn.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[mysql-asyn.sh\]: asyn test for mysql functionality +source $srcdir/diag.sh init +mysql --user=rsyslog --password=testbench < testsuites/mysql-truncate.sql +source $srcdir/diag.sh startup mysql-asyn.conf +source $srcdir/diag.sh injectmsg 0 50000 +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown +# note "-s" is requried to suppress the select "field header" +mysql -s --user=rsyslog --password=testbench < testsuites/mysql-select-msg.sql > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 49999 +source $srcdir/diag.sh exit diff --git a/tests/mysql-basic-vg.sh b/tests/mysql-basic-vg.sh new file mode 100755 index 00000000..215f41f0 --- /dev/null +++ b/tests/mysql-basic-vg.sh @@ -0,0 +1,14 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[mysql-basic-vg.sh\]: basic test for mysql-basic functionality/valgrind +source $srcdir/diag.sh init +mysql --user=rsyslog --password=testbench < testsuites/mysql-truncate.sql +source $srcdir/diag.sh startup-vg mysql-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +# note "-s" is requried to suppress the select "field header" +mysql -s --user=rsyslog --password=testbench < testsuites/mysql-select-msg.sql > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/mysql-basic.sh b/tests/mysql-basic.sh new file mode 100755 index 00000000..ba9d00f2 --- /dev/null +++ b/tests/mysql-basic.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[mysql-basic.sh\]: basic test for mysql-basic functionality +source $srcdir/diag.sh init +mysql --user=rsyslog --password=testbench < testsuites/mysql-truncate.sql +source $srcdir/diag.sh startup mysql-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown +# note "-s" is requried to suppress the select "field header" +mysql -s --user=rsyslog --password=testbench < testsuites/mysql-select-msg.sql > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/tcp-msgreduc-vg.sh b/tests/tcp-msgreduc-vg.sh index 7e388360..cd8534e4 100755 --- a/tests/tcp-msgreduc-vg.sh +++ b/tests/tcp-msgreduc-vg.sh @@ -12,5 +12,5 @@ source $srcdir/diag.sh wait-startup ./msleep 1500 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown-vg -source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg source $srcdir/diag.sh exit diff --git a/tests/testsuites/mysql-asyn.conf b/tests/testsuites/mysql-asyn.conf new file mode 100644 index 00000000..acdf9bbd --- /dev/null +++ b/tests/testsuites/mysql-asyn.conf @@ -0,0 +1,5 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/ommysql/.libs/ommysql +$ActionQueueType LinkedList +:msg, contains, "msgnum:" :ommysql:127.0.0.1,Syslog,rsyslog,testbench; diff --git a/tests/testsuites/mysql-basic.conf b/tests/testsuites/mysql-basic.conf new file mode 100644 index 00000000..070094f5 --- /dev/null +++ b/tests/testsuites/mysql-basic.conf @@ -0,0 +1,4 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/ommysql/.libs/ommysql +:msg, contains, "msgnum:" :ommysql:127.0.0.1,Syslog,rsyslog,testbench; diff --git a/tests/testsuites/mysql-select-msg.sql b/tests/testsuites/mysql-select-msg.sql new file mode 100644 index 00000000..2d27a331 --- /dev/null +++ b/tests/testsuites/mysql-select-msg.sql @@ -0,0 +1,2 @@ +use Syslog; +select substring(Message,9,8) from SystemEvents; diff --git a/tests/testsuites/mysql-truncate.sql b/tests/testsuites/mysql-truncate.sql new file mode 100644 index 00000000..ca852beb --- /dev/null +++ b/tests/testsuites/mysql-truncate.sql @@ -0,0 +1,2 @@ +use Syslog; +truncate table SystemEvents; -- cgit v1.2.3 From 8d79906d500c5760f093123b3ef3c43508ffce58 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 13:18:37 +0100 Subject: cosmetic: added bug info --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index c4bcadc2..784a8787 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,8 @@ Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? among others, life tests for ommysql (against a test database) have been added, valgrind-based testing enhanced, ... - bugfix: (regression) omhdfs did no longer compile +- bugfix: omlibdbi did not use password from rsyslog.con + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203 --------------------------------------------------------------------------- Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-09 - systemd support somewhat improved (can now take over existing log sockt) -- cgit v1.2.3 From 32acf5b9f7c706c1104e94e654fe8c934e7340ea Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 17:47:16 +0100 Subject: fixed problems in testbench makefile (just introduced...) --- tests/Makefile.am | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index acadda04..4a1bfe57 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -365,14 +365,14 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ imuxsock_ccmiddle_root.sh \ testsuites/imuxsock_ccmiddle_root.conf \ resultdata/imuxsock_ccmiddle.log \ - testsuites\mysql-truncate.sql \ - testsuites\mysql-select-msg.sql \ + testsuites/mysql-truncate.sql \ + testsuites/mysql-select-msg.sql \ mysql-basic.sh \ mysql-basic-vg.sh \ - testsuites\mysql-basic.sh \ + testsuites/mysql-basic.conf \ mysql-asyn.sh \ mysql-asyn-vg.sh \ - testsuites\mysql-asyn.sh \ + testsuites/mysql-asyn.conf \ cfg.sh ourtail_SOURCES = ourtail.c -- cgit v1.2.3 From 7fa1956da7ea3fa08d3187d3a5d883cfe5a67845 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Thu, 10 Mar 2011 03:23:58 +0100 Subject: bugfix: pmaixforwarded from fails to build Signed-off-by: Rainer Gerhards --- plugins/pmaixforwardedfrom/pmaixforwardedfrom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c index 11634199..fa4a9087 100644 --- a/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c +++ b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c @@ -41,6 +41,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.aixforwardedfrom") /* internal structures -- cgit v1.2.3 From 225aa0e367a604d3891bba13d470bfacc3dfe544 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Mar 2011 03:41:17 +0100 Subject: fixed problem in testbench Makefile, added module to distcheck Thanks to Michael Biebl for mentioning that some modules were not included in make distcheck. --- Makefile.am | 3 +++ tests/Makefile.am | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index f699cc46..4e3e9476 100644 --- a/Makefile.am +++ b/Makefile.am @@ -230,5 +230,8 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-impstats \ --enable-imptcp \ --enable-memcheck \ + --enable-pmaixforwardedfrom \ + --enable-pmcisconames \ + --enable-pmsnare \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 diff --git a/tests/Makefile.am b/tests/Makefile.am index 4a1bfe57..eb7ad94f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -6,7 +6,6 @@ TESTS = $(TESTRUNS) cfg.sh if ENABLE_IMDIAG TESTS += \ arrayqueue.sh \ - linkedlistqueue.sh \ da-mainmsg-q.sh \ validation-run.sh \ imtcp-multiport.sh \ @@ -54,7 +53,6 @@ TESTS += \ discard-allmark.sh \ discard.sh \ queue-persist.sh \ - arrayqueue.sh \ linkedlistqueue.sh endif @@ -214,7 +212,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rsf_getenv.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ + arrayqueue.sh \ testsuites/arrayqueue.conf \ + linkedlistqueue.sh \ testsuites/linkedlistqueue.conf \ da-mainmsg-q.sh \ testsuites/da-mainmsg-q.conf \ -- cgit v1.2.3 From 9cc963926bad3de9a78df51a19456f419c34492a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Mar 2011 14:46:47 +0100 Subject: bugfix: minor memory leak in omlibdbi (< 1k per instance and run) also testbench improvement (omlibdbi now also receives a couple of tests if we have a life MySQL server). --- ChangeLog | 1 + plugins/omlibdbi/omlibdbi.c | 9 ++++++++- tests/Makefile.am | 11 +++++++++++ tests/libdbi-asyn.sh | 13 +++++++++++++ tests/libdbi-basic-vg.sh | 16 ++++++++++++++++ tests/libdbi-basic.sh | 13 +++++++++++++ tests/testsuites/libdbi-asyn.conf | 12 ++++++++++++ tests/testsuites/libdbi-basic.conf | 9 +++++++++ 8 files changed, 83 insertions(+), 1 deletion(-) create mode 100755 tests/libdbi-asyn.sh create mode 100755 tests/libdbi-basic-vg.sh create mode 100755 tests/libdbi-basic.sh create mode 100644 tests/testsuites/libdbi-asyn.conf create mode 100644 tests/testsuites/libdbi-basic.conf diff --git a/ChangeLog b/ChangeLog index 784a8787..320134e4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,7 @@ Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? - improved testbench among others, life tests for ommysql (against a test database) have been added, valgrind-based testing enhanced, ... +- bugfix: minor memory leak in omlibdbi (< 1k per instance and run) - bugfix: (regression) omhdfs did no longer compile - bugfix: omlibdbi did not use password from rsyslog.con closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203 diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index f49bef42..7fcf9631 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -108,6 +108,11 @@ static void closeConn(instanceData *pData) BEGINfreeInstance CODESTARTfreeInstance closeConn(pData); + free(pData->drvrName); + free(pData->host); + free(pData->usrName); + free(pData->pwd); + free(pData->dbName); ENDfreeInstance @@ -171,7 +176,8 @@ static rsRetVal initConn(instanceData *pData, int bSilent) errmsg.LogError(0, RS_RET_SUSPENDED, "libdbi error: libdbi or libdbi drivers not present on this system - suspending."); ABORT_FINALIZE(RS_RET_SUSPENDED); } else if(iDrvrsLoaded < 0) { - errmsg.LogError(0, RS_RET_SUSPENDED, "libdbi error: libdbi could not be initialized - suspending."); + errmsg.LogError(0, RS_RET_SUSPENDED, "libdbi error: libdbi could not be " + "initialized (do you have any dbi drivers installed?) - suspending."); ABORT_FINALIZE(RS_RET_SUSPENDED); } bDbiInitialized = 1; /* we are done for the rest of our existence... */ @@ -367,6 +373,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr( (uchar *)"actionlibdbipassword", 0, eCmdHdlrGetWord, NULL, &pwd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr( (uchar *)"actionlibdbidbname", 0, eCmdHdlrGetWord, NULL, &dbName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr( (uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + DBGPRINTF("omlibdbi compiled with version %s loaded, libdbi version %s\n", VERSION, dbi_version()); ENDmodInit /* vim:set ai: diff --git a/tests/Makefile.am b/tests/Makefile.am index eb7ad94f..ea6860d5 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -63,10 +63,16 @@ TESTS += \ tcp-msgreduc-vg.sh endif + if ENABLE_MYSQL_TESTS TESTS += \ mysql-basic.sh \ mysql-asyn.sh +if ENABLE_OMLIBDBI +TESTS += \ + libdbi-basic.sh \ + libdbi-asyn.sh +endif if HAVE_VALGRIND TESTS += \ mysql-basic-vg.sh \ @@ -74,6 +80,7 @@ TESTS += \ endif endif + if ENABLE_IMPTCP TESTS += \ manyptcp.sh \ @@ -367,6 +374,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ resultdata/imuxsock_ccmiddle.log \ testsuites/mysql-truncate.sql \ testsuites/mysql-select-msg.sql \ + libdbi-basic.sh \ + testsuites/libdbi-basic.conf \ + libdbi-asyn.sh \ + testsuites/libdbi-asyn.conf \ mysql-basic.sh \ mysql-basic-vg.sh \ testsuites/mysql-basic.conf \ diff --git a/tests/libdbi-asyn.sh b/tests/libdbi-asyn.sh new file mode 100755 index 00000000..0076da9c --- /dev/null +++ b/tests/libdbi-asyn.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[libdbi-asyn.sh\]: asyn test for libdbi functionality +source $srcdir/diag.sh init +mysql --user=rsyslog --password=testbench < testsuites/mysql-truncate.sql +source $srcdir/diag.sh startup libdbi-asyn.conf +source $srcdir/diag.sh injectmsg 0 50000 +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown +# note "-s" is requried to suppress the select "field header" +mysql -s --user=rsyslog --password=testbench < testsuites/mysql-select-msg.sql > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 49999 +source $srcdir/diag.sh exit diff --git a/tests/libdbi-basic-vg.sh b/tests/libdbi-basic-vg.sh new file mode 100755 index 00000000..75d12857 --- /dev/null +++ b/tests/libdbi-basic-vg.sh @@ -0,0 +1,16 @@ +# This file is part of the rsyslog project, released under GPLv3 +# this test is currently not included in the testbench as libdbi +# itself seems to have a memory leak +echo =============================================================================== +echo \[libdbi-basic.sh\]: basic test for libdbi-basic functionality via mysql +source $srcdir/diag.sh init +mysql --user=rsyslog --password=testbench < testsuites/mysql-truncate.sql +source $srcdir/diag.sh startup-vg libdbi-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +# note "-s" is requried to suppress the select "field header" +mysql -s --user=rsyslog --password=testbench < testsuites/mysql-select-msg.sql > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/libdbi-basic.sh b/tests/libdbi-basic.sh new file mode 100755 index 00000000..a854a565 --- /dev/null +++ b/tests/libdbi-basic.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[libdbi-basic.sh\]: basic test for libdbi-basic functionality via mysql +source $srcdir/diag.sh init +mysql --user=rsyslog --password=testbench < testsuites/mysql-truncate.sql +source $srcdir/diag.sh startup libdbi-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown +# note "-s" is requried to suppress the select "field header" +mysql -s --user=rsyslog --password=testbench < testsuites/mysql-select-msg.sql > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/libdbi-asyn.conf b/tests/testsuites/libdbi-asyn.conf new file mode 100644 index 00000000..39b01fbc --- /dev/null +++ b/tests/testsuites/libdbi-asyn.conf @@ -0,0 +1,12 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/omlibdbi/.libs/omlibdbi + +$ActionQueueType LinkedList + +$ActionLibdbiDriver mysql +$ActionLibdbiHost 127.0.0.1 +$ActionLibdbiUserName root +$ActionLibdbiPassword pass +$ActionLibdbiDBName Syslog +:msg, contains, "msgnum:" :omlibdbi: diff --git a/tests/testsuites/libdbi-basic.conf b/tests/testsuites/libdbi-basic.conf new file mode 100644 index 00000000..212225cc --- /dev/null +++ b/tests/testsuites/libdbi-basic.conf @@ -0,0 +1,9 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/omlibdbi/.libs/omlibdbi +$ActionLibdbiDriver mysql +$ActionLibdbiHost 127.0.0.1 +$ActionLibdbiUserName root +$ActionLibdbiPassword pass +$ActionLibdbiDBName Syslog +:msg, contains, "msgnum:" :omlibdbi: -- cgit v1.2.3 From fd26a42bdc04eaf497cafd9ef806a54f3de1a7e9 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 10 Mar 2011 03:08:42 +0100 Subject: systemd: do not hook up SIGHUP for reload rsyslog doesn't actually do a configuration reload on SIGHUP, hence let's not pretend it does and drop the ExecReload= line, so that the clients will rather restart than reload the service. --- rsyslog.service.in | 1 - 1 file changed, 1 deletion(-) diff --git a/rsyslog.service.in b/rsyslog.service.in index 03db596e..898354d5 100644 --- a/rsyslog.service.in +++ b/rsyslog.service.in @@ -4,7 +4,6 @@ Description=System Logging Service [Service] ExecStartPre=/bin/systemctl stop systemd-kmsg-syslogd.service ExecStart=@sbindir@/rsyslogd -n -c5 -ExecReload=/bin/kill -HUP $MAINPID Sockets=syslog.socket [Install] -- cgit v1.2.3 From 6dbd70732a19a13bacae7b4bea07097458974cc9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Mar 2011 11:56:37 +0100 Subject: added one more test for imfile (under valgrind) --- tests/Makefile.am | 4 ++++ tests/imfile-basic-vg.sh | 15 +++++++++++++++ 2 files changed, 19 insertions(+) create mode 100755 tests/imfile-basic-vg.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index 9986cf80..c64d44e9 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -125,6 +125,9 @@ endif if ENABLE_IMFILE TESTS += imfile-basic.sh +if HAVE_VALGRIND +TESTS += imfile-basic-vg.sh +endif endif endif # if ENABLE_TESTBENCH @@ -327,6 +330,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ random.sh \ testsuites/random.conf \ imfile-basic.sh \ + imfile-basic-vg.sh \ testsuites/imfile-basic.conf \ dynfile_invld_async.sh \ dynfile_invld_sync.sh \ diff --git a/tests/imfile-basic-vg.sh b/tests/imfile-basic-vg.sh new file mode 100755 index 00000000..92cfc7f6 --- /dev/null +++ b/tests/imfile-basic-vg.sh @@ -0,0 +1,15 @@ +# This is part of the rsyslog testbench, licensed under GPLv3 +echo [imfile-basic.sh] +source $srcdir/diag.sh init +# generate input file first. Note that rsyslog processes it as +# soon as it start up (so the file should exist at that point). +./inputfilegen 50000 > rsyslog.input +ls -l rsyslog.input +source $srcdir/diag.sh startup-vg imfile-basic.conf +# sleep a little to give rsyslog a chance to begin processing +sleep 1 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +source $srcdir/diag.sh seq-check 0 49999 +source $srcdir/diag.sh exit -- cgit v1.2.3 From 4671cad51221f1cb03822481e46669ffba3c6d95 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Sun, 13 Mar 2011 12:09:06 +0100 Subject: Fall back to epoll_create() if epoll_create1() is not available epoll_create1() was introduced in Linux kernel 2.6.27. If rsyslog was compiled on a newer kernel but run on a kernel older than 2.6.27, remote syslog fails. Apply a runtime check for epoll_create1() and fall back to epoll_create() in this case. Bug-Debian: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=617996 --- plugins/imptcp/imptcp.c | 13 ++++++++----- plugins/imudp/imudp.c | 13 ++++++++----- runtime/nsdpoll_ptcp.c | 13 ++++++++----- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 3197564e..c4fbab3d 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -1047,17 +1047,20 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } -# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) - DBGPRINTF("imptcp uses epoll_create1()\n"); - epollfd = epoll_create1(EPOLL_CLOEXEC); -# else +#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imptcp uses epoll_create1()\n"); + epollfd = epoll_create1(EPOLL_CLOEXEC); + if(epollfd < 0 && errno ENOSYS) +#endif + { DBGPRINTF("imptcp uses epoll_create()\n"); /* reading the docs, the number of epoll events passed to * epoll_create() seems not to be used at all in kernels. So * we just provide "a" number, happens to be 10. */ epollfd = epoll_create(10); -# endif + } + if(epollfd < 0) { errmsg.LogError(0, RS_RET_EPOLL_CR_FAILED, "error: epoll_create() failed"); ABORT_FINALIZE(RS_RET_NO_RUN); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 56cdab28..a5002591 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -453,13 +453,16 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) CHKmalloc(udpEPollEvt = calloc(udpLstnSocks[0], sizeof(struct epoll_event))); -# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) - DBGPRINTF("imudp uses epoll_create1()\n"); - efd = epoll_create1(EPOLL_CLOEXEC); -# else +#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imudp uses epoll_create1()\n"); + efd = epoll_create1(EPOLL_CLOEXEC); + if(efd < 0 && errno == ENOSYS) +#endif + { DBGPRINTF("imudp uses epoll_create()\n"); efd = epoll_create(NUM_EPOLL_EVENTS); -# endif + } + if(efd < 0) { DBGPRINTF("epoll_create1() could not create fd\n"); ABORT_FINALIZE(RS_RET_IO_ERROR); diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c index bc374c60..ef9c37a3 100644 --- a/runtime/nsdpoll_ptcp.c +++ b/runtime/nsdpoll_ptcp.c @@ -133,13 +133,16 @@ delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) { /* Standard-Constructor */ BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */ -# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) - DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); - pThis->efd = epoll_create1(EPOLL_CLOEXEC); -# else +#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); + pThis->efd = epoll_create1(EPOLL_CLOEXEC); + if(pThis->efd < 0 && errno == ENOSYS) +#endif + { DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n"); pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */ -# endif + } + if(pThis->efd < 0) { DBGPRINTF("epoll_create1() could not create fd\n"); ABORT_FINALIZE(RS_RET_IO_ERROR); -- cgit v1.2.3 From 91d4e54a6707fa87298e367fb734fed931310b40 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 14 Mar 2011 09:24:57 +0100 Subject: fixed compile problem from epoll_create patch --- ChangeLog | 4 +++- plugins/imptcp/imptcp.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 320134e4..d5f855ae 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,9 +3,11 @@ Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? - improved testbench among others, life tests for ommysql (against a test database) have been added, valgrind-based testing enhanced, ... +- enhance: fallback *at runtime* to epoll_create if epoll_create1 is not + available. Thanks to Michael Biebl for analysis and patch! - bugfix: minor memory leak in omlibdbi (< 1k per instance and run) - bugfix: (regression) omhdfs did no longer compile -- bugfix: omlibdbi did not use password from rsyslog.con +- bugfix: omlibdbi did not use password from rsyslog.conf closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203 --------------------------------------------------------------------------- Version 5.7.8 [V5-BETA] (rgerhards), 2011-03-09 diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index c4fbab3d..e8cffbb0 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -1050,7 +1050,7 @@ CODESTARTwillRun #if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) DBGPRINTF("imptcp uses epoll_create1()\n"); epollfd = epoll_create1(EPOLL_CLOEXEC); - if(epollfd < 0 && errno ENOSYS) + if(epollfd < 0 && errno == ENOSYS) #endif { DBGPRINTF("imptcp uses epoll_create()\n"); -- cgit v1.2.3 From 33366f2f1e271f47edeed75f69d9885171e61ea0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 14 Mar 2011 09:36:37 +0100 Subject: added the template modules to the things being checked during distcheck While they are not meant for production, interface changes may not show up if they are not included in distecheck. This may be distrubing to users. The chances is even greater as there usually is no need to compile and/or check these modules during regular development. --- Makefile.am | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.am b/Makefile.am index 4e3e9476..53c04922 100644 --- a/Makefile.am +++ b/Makefile.am @@ -233,5 +233,7 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-pmaixforwardedfrom \ --enable-pmcisconames \ --enable-pmsnare \ + --enable-imtemplate \ + --enable-omtemplate \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 -- cgit v1.2.3 From 4a80242ffcfa93348453ff716f7f1abda417a416 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 16 Mar 2011 15:53:45 +0100 Subject: added some more valgrind-based tests --- tests/Makefile.am | 8 ++++++++ tests/failover-basic-vg.sh | 13 +++++++++++++ tests/failover-no-basic-vg.sh | 20 ++++++++++++++++++++ tests/failover-no-rptd-vg.sh | 20 ++++++++++++++++++++ tests/failover-rptd-vg.sh | 13 +++++++++++++ 5 files changed, 74 insertions(+) create mode 100755 tests/failover-basic-vg.sh create mode 100755 tests/failover-no-basic-vg.sh create mode 100755 tests/failover-no-rptd-vg.sh create mode 100755 tests/failover-rptd-vg.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index c1818f36..ebbe0cf6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -65,6 +65,10 @@ if HAVE_VALGRIND TESTS += \ discard-rptdmsg-vg.sh \ discard-allmark-vg.sh \ + failover-basic-vg.sh \ + failover-rptd-vg.sh \ + failover-no-basic-vg.sh \ + failover-no-rptd-vg.sh \ tcp-msgreduc-vg.sh endif @@ -265,12 +269,16 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ discard.sh \ testsuites/discard.conf \ failover-no-rptd.sh \ + failover-no-rptd-vg.sh \ testsuites/failover-no-rptd.conf \ failover-no-basic.sh \ + failover-no-basic-vg.sh \ testsuites/failover-no-basic.conf \ failover-rptd.sh \ + failover-rptd-vg.sh \ testsuites/failover-rptd.conf \ failover-basic.sh \ + failover-basic-vg.sh \ testsuites/failover-basic.conf \ discard-rptdmsg.sh \ discard-rptdmsg-vg.sh \ diff --git a/tests/failover-basic-vg.sh b/tests/failover-basic-vg.sh new file mode 100755 index 00000000..0eb77caa --- /dev/null +++ b/tests/failover-basic-vg.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-basic.sh\]: basic test for failover functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/failover-no-basic-vg.sh b/tests/failover-no-basic-vg.sh new file mode 100755 index 00000000..266163d4 --- /dev/null +++ b/tests/failover-no-basic-vg.sh @@ -0,0 +1,20 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-no-basic.sh\]: basic test for failover functionality - no failover +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-no-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +# now we need our custom logic to see if the result file is empty +# (what it should be!) +cmp rsyslog.out.log /dev/null +if [ $? -eq 1 ] +then + echo "ERROR, output file not empty" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/failover-no-rptd-vg.sh b/tests/failover-no-rptd-vg.sh new file mode 100755 index 00000000..3b77a0a4 --- /dev/null +++ b/tests/failover-no-rptd-vg.sh @@ -0,0 +1,20 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-no-rptd.sh\]: rptd test for failover functionality - no failover +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-no-rptd.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +# now we need our custom logic to see if the result file is empty +# (what it should be!) +cmp rsyslog.out.log /dev/null +if [ $? -eq 1 ] +then + echo "ERROR, output file not empty" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/failover-rptd-vg.sh b/tests/failover-rptd-vg.sh new file mode 100755 index 00000000..d208003b --- /dev/null +++ b/tests/failover-rptd-vg.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-rptd.sh\]: rptd test for failover functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-rptd.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit -- cgit v1.2.3 From 32db6291bc310062cffd3266fc297c7b13c571a7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 16 Mar 2011 16:59:49 +0100 Subject: backporting patch for clean make distcheck --- plugins/imdiag/imdiag.c | 3 +++ plugins/imfile/imfile.c | 8 ++++++++ runtime/cfsysline.c | 3 +++ 3 files changed, 14 insertions(+) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 0a69ee43..404cebc7 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -433,6 +433,9 @@ CODESTARTmodExit net.DestructPermittedPeers(&pPermPeersRoot); } + /* free some globals to keep valgrind happy */ + free(pszInputName); + /* release objects we used */ objRelease(net, LM_NET_FILENAME); objRelease(netstrm, LM_NETSTRMS_FILENAME); diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index acb58dad..cac3a55d 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -331,6 +331,11 @@ ENDrunInput */ BEGINwillRun CODESTARTwillRun + /* free config variables we do no longer needed */ + free(pszFileName); + free(pszFileTag); + free(pszStateFile); + if(iFilPtr == 0) { errmsg.LogError(0, RS_RET_NO_RUN, "No files configured to be monitored"); ABORT_FINALIZE(RS_RET_NO_RUN); @@ -398,6 +403,9 @@ CODESTARTafterRun persistStrmState(&files[i]); strm.Destruct(&(files[i].pStrm)); } + free(files[i].pszFileName); + free(files[i].pszTag); + free(files[i].pszStateFile); } if(pInputName != NULL) diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index 037e9f84..646ab39b 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -41,6 +41,7 @@ #include "obj.h" #include "errmsg.h" #include "srUtils.h" +#include "unicode-helper.h" /* static data */ @@ -511,6 +512,8 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); pStrB = NULL; + DBGPRINTF("doGetWord: get newval '%s' (len %d), hdlr %p\n", + pNewVal, (int) ustrlen(pNewVal), pSetHdlr); /* we got the word, now set it */ if(pSetHdlr == NULL) { /* we should set value directly to var */ -- cgit v1.2.3 From 709ce7647ca3a44c7c6d47e0445e70a9254a2fdb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 16 Mar 2011 17:17:34 +0100 Subject: preparing for 5.7.9 --- ChangeLog | 5 ++++- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 595a4585..13f6608d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,10 +1,13 @@ --------------------------------------------------------------------------- -Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? +Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-16 - improved testbench among others, life tests for ommysql (against a test database) have been added, valgrind-based testing enhanced, ... - enhance: fallback *at runtime* to epoll_create if epoll_create1 is not available. Thanks to Michael Biebl for analysis and patch! +- bugfix: failover did not work correctly if repeated msg reduction was on + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 + affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on - bugfix: minor memory leak in omlibdbi (< 1k per instance and run) - bugfix: (regression) omhdfs did no longer compile - bugfix: omlibdbi did not use password from rsyslog.conf diff --git a/configure.ac b/configure.ac index 58b203ff..4f368bcc 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.8],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.9],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 945a5a62..807ac0da 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.8 (beta branch) of rsyslog. +

    This documentation is for version 5.7.9 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 14f3857a0a122aa212b23a59175bd4e504176372 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Mar 2011 12:29:43 +0100 Subject: some minor enhancements to doc set -- point to more examples --- doc/rsyslog_conf.html | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 6990c6bd..703e7a6e 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -2,13 +2,16 @@ rsyslog.conf file

    rsyslog.conf configuration file

    -

    This document is currently being enhanced. Please -pardon its current appearance.

    Rsyslogd is configured via the rsyslog.conf file, typically found in /etc. By default, rsyslogd reads the file /etc/rsyslog.conf. This may be changed by a command line option.

    -Configuration file examples can be found in the rsyslog wiki.

    +Configuration file examples can be found in the rsyslog wiki. Also +keep the +rsyslog config snippets +on your mind. These are ready-to-use +real building blocks for rsyslog configuration. +

    There is also one sample file provided together with the documentation set. If you do not like to read, be sure to have at least a quick look at @@ -74,7 +77,7 @@ such features is available in rsyslogd, only.

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008,2009 by Rainer Gerhards and +Copyright © 2008-2011 by Rainer Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    -- cgit v1.2.3 From 68bde6252d22cc4e88edc2b51523726bb858c818 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Mar 2011 13:19:38 +0100 Subject: prepared skeleton for new strgen module --- ChangeLog | 2 + Makefile.am | 4 + configure.ac | 18 ++++ plugins/sm_cust_bindcdr/Makefile.am | 6 ++ plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 138 ++++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+) create mode 100644 plugins/sm_cust_bindcdr/Makefile.am create mode 100644 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c diff --git a/ChangeLog b/ChangeLog index 13f6608d..3503ed69 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ --------------------------------------------------------------------------- +Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-?? +--------------------------------------------------------------------------- Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-16 - improved testbench among others, life tests for ommysql (against a test database) have diff --git a/Makefile.am b/Makefile.am index 53c04922..2d73013b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -112,6 +112,10 @@ if ENABLE_CUST1 SUBDIRS += plugins/cust1 endif +if ENABLE_SMCUSTBINDCDR +SUBDIRS += plugins/sm_cust_bindcdr +endif + if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif diff --git a/configure.ac b/configure.ac index 4f368bcc..832a5b20 100644 --- a/configure.ac +++ b/configure.ac @@ -1081,6 +1081,20 @@ AC_ARG_ENABLE(cust1, AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes) +# A custom strgen that also serves as a sample of how to do +# SQL-generating strgen's +AC_ARG_ENABLE(smcustbindcdr, + [AS_HELP_STRING([--enable-smcustbindcdr],[Compiles smcustbindcdr module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_smcustbindcdr="yes" ;; + no) enable_smcustbindcdr="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-smcustbindcdr) ;; + esac], + [enable_smcustbindcdr=no] +) +AM_CONDITIONAL(ENABLE_SMCUSTBINDCDR, test x$enable_smcustbindcdr = xyes) + + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -1178,6 +1192,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ plugins/omudpspoof/Makefile \ + plugins/sm_cust_bindcdr/Makefile \ plugins/cust1/Makefile \ java/Makefile \ tests/Makefile]) @@ -1223,6 +1238,9 @@ echo " pmcisconames module will be compiled: $enable_pmcisconames" echo " pmaixforwardedfrom module w.be compiled: $enable_pmaixforwardedfrom" echo " pmsnare module will be compiled: $enable_pmsnare" echo +echo "---{ strgen modules }---" +echo " sm_cust_bindcdr module will be compiled: $enable_sm_cust_bindcdr" +echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" echo " libdbi support enabled: $enable_libdbi" diff --git a/plugins/sm_cust_bindcdr/Makefile.am b/plugins/sm_cust_bindcdr/Makefile.am new file mode 100644 index 00000000..1f71d499 --- /dev/null +++ b/plugins/sm_cust_bindcdr/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = sm_cust_bindcdr.la + +sm_cust_bindcdr_la_SOURCES = sm_cust_bindcdr.c +sm_cust_bindcdr_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +sm_cust_bindcdr_la_LDFLAGS = -module -avoid-version +sm_cust_bindcdr_la_LIBADD = diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c new file mode 100644 index 00000000..cf3cdeef --- /dev/null +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -0,0 +1,138 @@ +/* sm_cust_bindcdr.c + * This is a custom developed plugin to process bind information into + * a specific SQL statement. While the actual processing may be too specific + * to be of general use, this module serves as a template on how this type + * of processing can be done. + * + * Format generated: + * "%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" + * Note that this is the same as smtradfile.c, except that we do have a RFC3339 timestamp. However, + * we have copied over the code from there, it is too simple to go through all the hassle + * of having a single code base. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2011-03-17 by RGerhards + * + * Copyright 2011 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" + +MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP +STRGEN_NAME("Custom_BindCDR") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA + + +/* config data */ + + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + */ +BEGINstrgen + register int iBuf; + uchar *pTimeStamp; + size_t lenTimeStamp; + uchar *pHOSTNAME; + size_t lenHOSTNAME; + uchar *pTAG; + int lenTAG; + uchar *pMSG; + size_t lenMSG; + size_t lenTotal; +CODESTARTstrgen + /* first obtain all strings and their length (if not fixed) */ + pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); + lenTimeStamp = ustrlen(pTimeStamp); + pHOSTNAME = (uchar*) getHOSTNAME(pMsg); + lenHOSTNAME = getHOSTNAMELen(pMsg); + getTAG(pMsg, &pTAG, &lenTAG); + pMSG = getMSG(pMsg); + lenMSG = getMSGLen(pMsg); + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2; + if(pMSG[0] != ' ') + ++lenTotal; /* then we need to introduce one additional space */ + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + memcpy(*ppBuf, pTimeStamp, lenTimeStamp); + iBuf = lenTimeStamp; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME); + iBuf += lenHOSTNAME; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pTAG, lenTAG); + iBuf += lenTAG; + + if(pMSG[0] != ' ') + *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf + iBuf, pMSG, lenMSG); + iBuf += lenMSG; + + /* trailer */ + *(*ppBuf + iBuf++) = '\n'; + *(*ppBuf + iBuf) = '\0'; + +finalize_it: +ENDstrgen + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(sm_cust_bindcdr) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + + dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION); +ENDmodInit -- cgit v1.2.3 From 459f0d094fcb48fd58204ad3527f931715839bcb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Mar 2011 14:59:18 +0100 Subject: setup test environment and test code in order to look at... bug http://bugzilla.adiscon.com/show_bug.cgi?id=195 That bug currently prevents strgen's to be used with databases --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 45 ++++++++++--------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index cf3cdeef..5ea7e653 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -63,56 +63,39 @@ DEF_SMOD_STATIC_DATA * needed (including their length) and then calculating the actual space required. So when we * finally copy, we know exactly what we need. So we do at most one alloc. */ +//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" +#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('" BEGINstrgen register int iBuf; uchar *pTimeStamp; size_t lenTimeStamp; - uchar *pHOSTNAME; - size_t lenHOSTNAME; - uchar *pTAG; - int lenTAG; - uchar *pMSG; - size_t lenMSG; size_t lenTotal; CODESTARTstrgen /* first obtain all strings and their length (if not fixed) */ pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); lenTimeStamp = ustrlen(pTimeStamp); - pHOSTNAME = (uchar*) getHOSTNAME(pMsg); - lenHOSTNAME = getHOSTNAMELen(pMsg); - getTAG(pMsg, &pTAG, &lenTAG); - pMSG = getMSG(pMsg); - lenMSG = getMSGLen(pMsg); /* calculate len, constants for spaces and similar fixed strings */ - lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2; - if(pMSG[0] != ' ') - ++lenTotal; /* then we need to introduce one additional space */ + lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2; /* now make sure buffer is large enough */ if(lenTotal >= *pLenBuf) CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); /* and concatenate the resulting string */ - memcpy(*ppBuf, pTimeStamp, lenTimeStamp); - iBuf = lenTimeStamp; - *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1); + iBuf = sizeof(SQL_STMT) - 1; - memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME); - iBuf += lenHOSTNAME; - *(*ppBuf + iBuf++) = ' '; + // SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); - memcpy(*ppBuf + iBuf, pTAG, lenTAG); - iBuf += lenTAG; + memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp); + iBuf += lenTimeStamp; + memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1); + iBuf += sizeof("', '") - 1; - if(pMSG[0] != ' ') - *(*ppBuf + iBuf++) = ' '; - memcpy(*ppBuf + iBuf, pMSG, lenMSG); - iBuf += lenMSG; - - /* trailer */ - *(*ppBuf + iBuf++) = '\n'; - *(*ppBuf + iBuf) = '\0'; + /* end of SQL statement/trailer (NUL is contained in string!) */ + memcpy(*ppBuf + iBuf, "');", sizeof("');")); + iBuf += sizeof("');"); finalize_it: ENDstrgen @@ -129,7 +112,7 @@ CODEqueryEtryPt_STD_SMOD_QUERIES ENDqueryEtryPt -BEGINmodInit(sm_cust_bindcdr) +BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr -- cgit v1.2.3 From e2f13df22c7599fa1fc7d6b4c20081aac78b355a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Mar 2011 09:53:55 +0100 Subject: potential fix to issue that strgen's do not support SQL option needs testig and verification (wrong system for doing that ;)) --- template.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/template.c b/template.c index 5aaf1bcb..cc339653 100644 --- a/template.c +++ b/template.c @@ -856,19 +856,37 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) { uchar *pSrc, *pDst; uchar szMod[2048]; + unsigned lenMod; strgen_t *pStrgen; DEFiRet; pSrc = *ppRestOfConfLine; pDst = szMod; - while(*pSrc && !isspace(*pSrc) && pDst < &(szMod[sizeof(szMod) - 1])) { - *pDst++ = *pSrc++; + lenMod = 0; + while(*pSrc && !isspace(*pSrc) && lenMod < sizeof(szMod) - 1) { + szMod[lenMod] = *pSrc++; + lenMod++; + } - *pDst = '\0'; + szMod[lenMod] = '\0'; *ppRestOfConfLine = pSrc; CHKiRet(strgen.FindStrgen(&pStrgen, szMod)); pTpl->pStrgen = pStrgen->pModule->mod.sm.strgen; - dbgprintf("template bound to strgen '%s'\n", szMod); + DBGPRINTF("template bound to strgen '%s'\n", szMod); + /* check if the name potentially contains some well-known options + * Note: we have opted to let the name contain all options. This sounds + * useful, because the strgen MUST actually implement a specific set + * of options. Doing this via the name looks to the enduser as if the + * regular syntax were used, and it make sure the strgen postively + * acknowledged implementing the option. -- rgerhards, 2011-03-21 + */ + if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 4, ",stdsql")) { + pTpl->optFormatForSQL = 2; + DBGPRINTF("strgen suports the stdsql option\n"); + } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 7, ",sql")) { + pTpl->optFormatForSQL = 1; + DBGPRINTF("strgen suports the sql option\n"); + } finalize_it: RETiRet; @@ -936,8 +954,7 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) /* we simply make the template defunct in this case by setting * its name to a zero-string. We do not free it, as this would * require additional code and causes only a very small memory - * consumption. Memory is freed, however, in normal operation - * and most importantly by HUPing syslogd. + * consumption. */ *pTpl->pszName = '\0'; return NULL; -- cgit v1.2.3 From 41d79b8fead00f85614547606b0c1117f3efa04e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Mar 2011 10:34:43 +0100 Subject: bugfix: strgen could not be used together with database outputs because the sql/stdsql option could not be specified. This has been solved by permitting the strgen to include the opton inside its name. closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195 --- ChangeLog | 6 ++++++ plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 2 +- template.c | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3503ed69..70d7d322 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,11 @@ --------------------------------------------------------------------------- Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-?? +- new sample plugin for a strgen to generate sql statement consumable + by a database plugin +- bugfix: strgen could not be used together with database outputs + because the sql/stdsql option could not be specified. This has been + solved by permitting the strgen to include the opton inside its name. + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195 --------------------------------------------------------------------------- Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-16 - improved testbench diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index 5ea7e653..d1634252 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -49,7 +49,7 @@ MODULE_TYPE_STRGEN MODULE_TYPE_NOKEEP -STRGEN_NAME("Custom_BindCDR") +STRGEN_NAME("Custom_BindCDR,sql") /* internal structures */ diff --git a/template.c b/template.c index cc339653..3f6942c2 100644 --- a/template.c +++ b/template.c @@ -880,10 +880,10 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) * regular syntax were used, and it make sure the strgen postively * acknowledged implementing the option. -- rgerhards, 2011-03-21 */ - if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 4, ",stdsql")) { + if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) { pTpl->optFormatForSQL = 2; DBGPRINTF("strgen suports the stdsql option\n"); - } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 7, ",sql")) { + } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) { pTpl->optFormatForSQL = 1; DBGPRINTF("strgen suports the sql option\n"); } -- cgit v1.2.3 From 5808af91e7e5e552a5cde4603687685978934626 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Mar 2011 12:17:20 +0100 Subject: added internal processing of BIND records still some questions outstanding, so actual SQL statement is not usable at the moment --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 121 ++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 6 deletions(-) diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index d1634252..be01db77 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -37,6 +37,7 @@ #include "config.h" #include "rsyslog.h" #include +#include #include #include #include @@ -64,19 +65,112 @@ DEF_SMOD_STATIC_DATA * finally copy, we know exactly what we need. So we do at most one alloc. */ //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" -#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('" +//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('" +#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('" +#define ADD_SQL_DELIM \ + memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \ + iBuf += sizeof("', '") - 1; +#define SQL_STMT_END "');\n" BEGINstrgen register int iBuf; + uchar *psz; uchar *pTimeStamp; + uchar szClient[64]; + unsigned lenClient; + uchar szView[64]; + unsigned lenView; + uchar szQuery[64]; + unsigned lenQuery; + uchar szIP[64]; + unsigned lenIP; size_t lenTimeStamp; size_t lenTotal; CODESTARTstrgen + /* first create an empty statement. This is to be replaced if + * we have better data to fill in. + */ + /* now make sure buffer is large enough */ + if(*pLenBuf < 2) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, 2)); + memcpy(*ppBuf, ";", sizeof(";")); + /* first obtain all strings and their length (if not fixed) */ pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); lenTimeStamp = ustrlen(pTimeStamp); + + /* "client" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: client part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("client ") - 1; /* skip "label" */ + for( lenClient = 0 + ; *psz && *psz != '#' && lenClient < sizeof(szClient) - 1 + ; ++lenClient) { + szClient[lenClient] = *psz++; + } + szClient[lenClient] = '\0'; + } + + /* "view" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "view "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: view part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("view ") - 1; /* skip "label" */ + for( lenView = 0 + ; *psz && *psz != ':' && lenView < sizeof(szView) - 1 + ; ++lenView) { + szView[lenView] = *psz++; + } + szView[lenView] = '\0'; + } + + /* "query" - we must extract just the number, and in reverse! */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "query: "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: query part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("query: ") - 1; /* skip "label" */ + /* first find end-of-string to process */ + while(*psz && (isdigit(*psz) || *psz == '.')) { +dbgprintf("XXXX: step 1: %c\n", *psz); + psz++; + } + /* now shuffle data */ + for( lenQuery = 0 + ; *psz && *psz != ' ' && lenQuery < sizeof(szQuery) - 1 + ; --psz) { + if(isdigit(*psz)) + szQuery[lenQuery++] = *psz; + } + szQuery[lenQuery] = '\0'; + } + + /* "ip" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + ("); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: ip part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("IN TXT + (") - 1; /* skip "label" */ + for( lenIP = 0 + ; *psz && *psz != ')' && lenIP < sizeof(szIP) - 1 + ; ++lenIP) { + szIP[lenIP] = *psz++; + } + szIP[lenIP] = '\0'; + } + + + /* --- strings extracted ---- */ /* calculate len, constants for spaces and similar fixed strings */ - lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2; + lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5 + + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; /* now make sure buffer is large enough */ if(lenTotal >= *pLenBuf) @@ -90,12 +184,27 @@ CODESTARTstrgen memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp); iBuf += lenTimeStamp; - memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1); - iBuf += sizeof("', '") - 1; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szClient, lenClient); + iBuf += lenClient; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szView, lenView); + iBuf += lenView; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szQuery, lenQuery); + iBuf += lenQuery; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szIP, lenIP); + iBuf += lenIP; + ADD_SQL_DELIM /* end of SQL statement/trailer (NUL is contained in string!) */ - memcpy(*ppBuf + iBuf, "');", sizeof("');")); - iBuf += sizeof("');"); + memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END)); + iBuf += sizeof(SQL_STMT_END); finalize_it: ENDstrgen -- cgit v1.2.3 From 0d80f4e4612fb209cc9dda5faf7b71cd261c0c20 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Mar 2011 14:22:38 +0100 Subject: sm_bind_cdr: added capability to configure "allowed IPs" --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 83 +++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index be01db77..005a4ba9 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -43,10 +43,12 @@ #include #include "conf.h" #include "syslogd-types.h" +#include "cfsysline.h" #include "template.h" #include "msg.h" #include "module-template.h" #include "unicode-helper.h" +#include "errmsg.h" MODULE_TYPE_STRGEN MODULE_TYPE_NOKEEP @@ -55,10 +57,65 @@ STRGEN_NAME("Custom_BindCDR,sql") /* internal structures */ DEF_SMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +/* list of "allowed" IPs */ +typedef struct allowedip_s { + uchar *pszIP; + struct allowedip_s *next; +} allowedip_t; + +static allowedip_t *root; /* config data */ +/* check if the provided IP is (already) in the allowed list + */ +static int +isAllowed(uchar *pszIP) +{ + allowedip_t *pallow; + int ret = 0; + + for(pallow = root ; pallow != NULL ; pallow = pallow->next) { + DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP); + if(!ustrcmp(pallow->pszIP, pszIP)) { + ret = 1; + goto finalize_it; + } + } +finalize_it: return ret; +} + +/* This function is called to add an additional allowed IP. It adds + * the IP to the linked list of them. An error is emitted if the IP + * already exists. + */ +static rsRetVal addAllowedIP(void __attribute__((unused)) *pVal, uchar *pNewVal) +{ + allowedip_t *pNew; + DEFiRet; + + if(isAllowed(pNewVal)) { + errmsg.LogError(0, NO_ERRCODE, "error: allowed IP '%s' already configured " + "duplicate ignored", pNewVal); + ABORT_FINALIZE(RS_RET_ERR); + } + + CHKmalloc(pNew = malloc(sizeof(allowedip_t))); + pNew->pszIP = pNewVal; + pNew->next = root; + root = pNew; + DBGPRINTF("sm_cust_bindcdr: allowed IP '%s' added.\n", pNewVal); + +finalize_it: + if(iRet != RS_RET_OK) { + free(pNewVal); + } + + RETiRet; +} /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings * needed (including their length) and then calculating the actual space required. So when we @@ -72,9 +129,10 @@ DEF_SMOD_STATIC_DATA iBuf += sizeof("', '") - 1; #define SQL_STMT_END "');\n" BEGINstrgen - register int iBuf; + int iBuf; uchar *psz; uchar *pTimeStamp; + size_t lenTimeStamp; uchar szClient[64]; unsigned lenClient; uchar szView[64]; @@ -83,7 +141,6 @@ BEGINstrgen unsigned lenQuery; uchar szIP[64]; unsigned lenIP; - size_t lenTimeStamp; size_t lenTotal; CODESTARTstrgen /* first create an empty statement. This is to be replaced if @@ -137,7 +194,6 @@ CODESTARTstrgen psz += sizeof("query: ") - 1; /* skip "label" */ /* first find end-of-string to process */ while(*psz && (isdigit(*psz) || *psz == '.')) { -dbgprintf("XXXX: step 1: %c\n", *psz); psz++; } /* now shuffle data */ @@ -168,6 +224,14 @@ dbgprintf("XXXX: step 1: %c\n", *psz); /* --- strings extracted ---- */ + /* now check if the IP is "allowed", in which case we should not + * insert into the database. + */ + if(isAllowed(szIP)) { + DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n"); + FINALIZE; + } + /* calculate len, constants for spaces and similar fixed strings */ lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; @@ -211,7 +275,16 @@ ENDstrgen BEGINmodExit + allowedip_t *pallow, *pdel; CODESTARTmodExit + for(pallow = root ; pallow != NULL ; ) { + pdel = pallow; + pallow = pallow->next; + free(pdel->pszIP); + free(pdel); + } + + objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -225,6 +298,10 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + root = NULL; + CHKiRet(omsdRegCFSLineHdlr((uchar *)"sgcustombindcdrallowedip", 0, eCmdHdlrGetWord, + addAllowedIP, NULL, STD_LOADABLE_MODULE_ID)); dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION); ENDmodInit -- cgit v1.2.3 From 27ab3768e2eaefaed7054144939f0730b3fac84f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Mar 2011 15:02:15 +0100 Subject: bugfix: RFC5424 parser confused by empty structured data closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237 --- ChangeLog | 2 ++ tools/pmrfc5424.c | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 13f6608d..de55c799 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- bugfix: RFC5424 parser confused by empty structured data + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237 --------------------------------------------------------------------------- Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-16 - improved testbench diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c index 2bd18042..b06f1347 100644 --- a/tools/pmrfc5424.c +++ b/tools/pmrfc5424.c @@ -142,7 +142,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr * structured data. There may also be \] inside the structured data, which * do NOT terminate an element. */ - if(lenStr == 0 || *p2parse != '[') + if(lenStr == 0 || (*p2parse != '[' && *p2parse != '-')) return 1; /* this is NOT structured data! */ if(*p2parse == '-') { /* empty structured data? */ @@ -211,7 +211,6 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr * * rger, 2005-11-24 */ -//static int parseRFCSyslogMsg(msg_t *pMsg, int flags) BEGINparse uchar *p2parse; uchar *pBuf = NULL; -- cgit v1.2.3 From e69efbb6d7907339ef81a2062f858eb082233f61 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Mar 2011 16:55:11 +0100 Subject: enhance: added $BOM system property to ease writing byte order masks --- ChangeLog | 1 + doc/property_replacer.html | 6 ++++++ runtime/msg.c | 10 ++++++++++ runtime/rsyslog.h | 1 + 4 files changed, 18 insertions(+) diff --git a/ChangeLog b/ChangeLog index de55c799..846d2850 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +- enhance: added $BOM system property to ease writing byte order masks - bugfix: RFC5424 parser confused by empty structured data closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237 --------------------------------------------------------------------------- diff --git a/doc/property_replacer.html b/doc/property_replacer.html index 4d242a34..cd357f67 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -156,6 +156,12 @@ than messages generated somewhere. +$bom +The UTF-8 encoded Unicode byte-order mask (BOM). This may be useful +in templates for RFC5424 support, when the character set is know to be +Unicode. + + $now The current date stamp in the format YYYY-MM-DD diff --git a/runtime/msg.c b/runtime/msg.c index b0261faa..e8e60963 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -444,6 +444,8 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) *pPropID = PROP_SYS_MINUTE; } else if(!strcmp((char*) pName, "$myhostname")) { *pPropID = PROP_SYS_MYHOSTNAME; + } else if(!strcmp((char*) pName, "$bom")) { + *pPropID = PROP_SYS_BOM; } else { *pPropID = PROP_INVALID; iRet = RS_RET_VAR_NOT_FOUND; @@ -525,6 +527,8 @@ uchar *propIDToName(propid_t propID) return UCHAR_CONSTANT("$MINUTE"); case PROP_SYS_MYHOSTNAME: return UCHAR_CONSTANT("$MYHOSTNAME"); + case PROP_SYS_BOM: + return UCHAR_CONSTANT("$BOM"); default: return UCHAR_CONSTANT("*invalid property id*"); } @@ -2427,6 +2431,12 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, case PROP_SYS_MYHOSTNAME: pRes = glbl.GetLocalHostName(); break; + case PROP_SYS_BOM: + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (uchar*) "\xEF\xBB\xBF"; + *pbMustBeFreed = 0; + break; default: /* there is no point in continuing, we may even otherwise render the * error message unreadable. rgerhards, 2007-07-10 diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 78e61bcb..d63dbe4f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -137,6 +137,7 @@ typedef uintTiny propid_t; #define PROP_SYS_QHOUR 156 #define PROP_SYS_MINUTE 157 #define PROP_SYS_MYHOSTNAME 158 +#define PROP_SYS_BOM 159 /* The error codes below are orginally "borrowed" from -- cgit v1.2.3 From eefc9b6eb69c130015955cfb15dc45e36e431d5a Mon Sep 17 00:00:00 2001 From: Corey Smith Date: Thu, 24 Mar 2011 17:25:03 +0100 Subject: bugfix: PRI was invalid on Solaris for message from local log socket Signed-off-by: root --- ChangeLog | 1 + runtime/parser.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 846d2850..dc6b9872 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +- bugfix: PRI was invalid on Solaris for message from local log socket - enhance: added $BOM system property to ease writing byte order masks - bugfix: RFC5424 parser confused by empty structured data closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237 diff --git a/runtime/parser.c b/runtime/parser.c index d3644976..b385c54b 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -453,10 +453,10 @@ ParsePRI(msg_t *pMsg) if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) pri = DEFUPRI; } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); } - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); - MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); RETiRet; } -- cgit v1.2.3 From 0fcbbfbe3b40e2739531cfa3a308c6fc8681ee7f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 28 Mar 2011 16:09:07 +0200 Subject: sm_cust_bindcdr: custom date parsing added --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 100 +++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 10 deletions(-) diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index 005a4ba9..fa16acb5 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -120,6 +120,8 @@ finalize_it: /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings * needed (including their length) and then calculating the actual space required. So when we * finally copy, we know exactly what we need. So we do at most one alloc. + * An actual message sample for what we intend to parse is (one line): + <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96) */ //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" //#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('" @@ -131,8 +133,12 @@ finalize_it: BEGINstrgen int iBuf; uchar *psz; - uchar *pTimeStamp; - size_t lenTimeStamp; + uchar szDate[64]; + unsigned lenDate; + uchar szTime[64]; + unsigned lenTime; + uchar szMSec[64]; + unsigned lenMSec; uchar szClient[64]; unsigned lenClient; uchar szView[64]; @@ -152,9 +158,76 @@ CODESTARTstrgen memcpy(*ppBuf, ";", sizeof(";")); /* first obtain all strings and their length (if not fixed) */ - pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); - lenTimeStamp = ustrlen(pTimeStamp); - + /* Note that there are two date fields present, one in the header + * and one more in the actual message. We use the one from the message + * and parse that our. We check validity based on some fixe fields. In- + * depth verification is probably not worth the effort (CPU time), because + * we do various other checks on the message format below). + */ + psz = getMSG(pMsg); + if(psz[0] == ' ' && psz[3] == '-' && psz[7] == '-') { + memcpy(szDate, psz+8, 4); + szDate[4] = '-'; + if(!strncmp((char*)psz+4, "Jan", 3)) { + szDate[5] = '0'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Feb", 3)) { + szDate[5] = '0'; + szDate[6] = '2'; + } else if(!strncmp((char*)psz+4, "Mar", 3)) { + szDate[5] = '0'; + szDate[6] = '3'; + } else if(!strncmp((char*)psz+4, "Apr", 3)) { + szDate[5] = '0'; + szDate[6] = '4'; + } else if(!strncmp((char*)psz+4, "May", 3)) { + szDate[5] = '0'; + szDate[6] = '5'; + } else if(!strncmp((char*)psz+4, "Jun", 3)) { + szDate[5] = '0'; + szDate[6] = '6'; + } else if(!strncmp((char*)psz+4, "Jul", 3)) { + szDate[5] = '0'; + szDate[6] = '7'; + } else if(!strncmp((char*)psz+4, "Aug", 3)) { + szDate[5] = '0'; + szDate[6] = '8'; + } else if(!strncmp((char*)psz+4, "Sep", 3)) { + szDate[5] = '0'; + szDate[6] = '9'; + } else if(!strncmp((char*)psz+4, "Oct", 3)) { + szDate[5] = '1'; + szDate[6] = '0'; + } else if(!strncmp((char*)psz+4, "Nov", 3)) { + szDate[5] = '1'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Dec", 3)) { + szDate[5] = '1'; + szDate[6] = '2'; + } + szDate[7] = '-'; + szDate[8] = psz[1]; + szDate[9] = psz[2]; + szDate[10] = '\0'; + lenDate = 10; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + FINALIZE; + } + + /* now time (pull both regular time and ms) */ + if(psz[12] == ' ' && psz[15] == ':' && psz[18] == ':' && psz[21] == '.' && psz[25] == ' ') { + memcpy(szTime, (char*)psz+13, 8); + szTime[9] = '\0'; + lenTime = 8; + memcpy(szMSec, (char*)psz+22, 3); + szMSec[4] = '\0'; + lenMSec = 3; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + FINALIZE; + } + /* "client" */ psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); if(psz == NULL) { @@ -233,8 +306,8 @@ CODESTARTstrgen } /* calculate len, constants for spaces and similar fixed strings */ - lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5 - + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; + lenTotal = lenDate + lenTime + lenMSec + lenClient + lenView + lenQuery + + lenIP + 7 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; /* now make sure buffer is large enough */ if(lenTotal >= *pLenBuf) @@ -246,8 +319,16 @@ CODESTARTstrgen // SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); - memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp); - iBuf += lenTimeStamp; + memcpy(*ppBuf + iBuf, szDate, lenDate); + iBuf += lenDate; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szTime, lenTime); + iBuf += lenTime; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szMSec, lenMSec); + iBuf += lenMSec; ADD_SQL_DELIM memcpy(*ppBuf + iBuf, szClient, lenClient); @@ -264,7 +345,6 @@ CODESTARTstrgen memcpy(*ppBuf + iBuf, szIP, lenIP); iBuf += lenIP; - ADD_SQL_DELIM /* end of SQL statement/trailer (NUL is contained in string!) */ memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END)); -- cgit v1.2.3 From 6f9773f1faa79c87c4c35c679455151d4e6d9e00 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 28 Mar 2011 17:40:15 +0200 Subject: bugfix: rsyslog did not build with --disable-regexp configure option closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243 --- ChangeLog | 2 ++ template.c | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index dc6b9872..cbc91b87 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- bugfix: rsyslog did not build with --disable-regexp configure option + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243 - bugfix: PRI was invalid on Solaris for message from local log socket - enhance: added $BOM system property to ease writing byte order masks - bugfix: RFC5424 parser confused by empty structured data diff --git a/template.c b/template.c index cc339653..1e1a6c3f 100644 --- a/template.c +++ b/template.c @@ -42,10 +42,13 @@ /* static data */ DEFobjCurrIf(obj) DEFobjCurrIf(errmsg) -DEFobjCurrIf(regexp) DEFobjCurrIf(strgen) +#ifdef FEATURE_REGEXP +DEFobjCurrIf(regexp) static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */ +#endif + static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ @@ -549,10 +552,9 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) cstr_t *pStrB; struct templateEntry *pTpe; int iNum; /* to compute numbers */ - rsRetVal iRetLocal; - #ifdef FEATURE_REGEXP /* APR: variables for regex */ + rsRetVal iRetLocal; int longitud; unsigned char *regex_char; unsigned char *regex_end; @@ -1084,11 +1086,13 @@ void tplDeleteAll(void) break; case FIELD: /* check if we have a regexp and, if so, delete it */ +#ifdef FEATURE_REGEXP if(pTpeDel->data.field.has_regex != 0) { if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { regexp.regfree(&(pTpeDel->data.field.re)); } } +#endif break; } /*dbgprintf("\n");*/ @@ -1137,12 +1141,14 @@ void tplDeleteNew(void) free(pTpeDel->data.constant.pConstant); break; case FIELD: +#ifdef FEATURE_REGEXP /* check if we have a regexp and, if so, delete it */ if(pTpeDel->data.field.has_regex != 0) { if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { regexp.regfree(&(pTpeDel->data.field.re)); } } +#endif break; } /*dbgprintf("\n");*/ -- cgit v1.2.3 From d2cce9852bdd9ba9f93ac94d6c8b0178d5b82bb0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 29 Mar 2011 11:48:45 +0200 Subject: bugfix: error return from strgen caused abort, now causes action to be ignored ...(just like a failed filter) --- ChangeLog | 2 ++ action.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 70d7d322..7d868d64 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-?? +- bugfix: error return from strgen caused abort, now causes action to be + ignored (just like a failed filter) - new sample plugin for a strgen to generate sql statement consumable by a database plugin - bugfix: strgen could not be used together with database outputs diff --git a/action.c b/action.c index ea4358fb..c5bd03cb 100644 --- a/action.c +++ b/action.c @@ -1071,7 +1071,8 @@ prepareBatch(action_t *pAction, batch_t *pBatch) pElem = &(pBatch->pElem[i]); if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) { pElem->state = BATCH_STATE_RDY; - prepareDoActionParams(pAction, pElem); + if(prepareDoActionParams(pAction, pElem) != RS_RET_OK) + pElem->bFilterOK = FALSE; } } RETiRet; -- cgit v1.2.3 From 030fd145a3d8759367b73cc66e03023ff131c951 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 29 Mar 2011 11:50:32 +0200 Subject: did mapping of data items to database table columns to facilitate testing --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index fa16acb5..3fe96ac4 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -79,7 +79,6 @@ isAllowed(uchar *pszIP) int ret = 0; for(pallow = root ; pallow != NULL ; pallow = pallow->next) { - DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP); if(!ustrcmp(pallow->pszIP, pszIP)) { ret = 1; goto finalize_it; @@ -123,9 +122,7 @@ finalize_it: * An actual message sample for what we intend to parse is (one line): <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96) */ -//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" -//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('" -#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('" +#define SQL_STMT "INSERT INTO CDR(`Date`,`Time`, timeMS, client, view, query, ip) VALUES ('" #define ADD_SQL_DELIM \ memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \ iBuf += sizeof("', '") - 1; @@ -212,7 +209,7 @@ CODESTARTstrgen lenDate = 10; } else { dbgprintf("Custom_BindCDR: date part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } /* now time (pull both regular time and ms) */ @@ -225,14 +222,14 @@ CODESTARTstrgen lenMSec = 3; } else { dbgprintf("Custom_BindCDR: date part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } /* "client" */ psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); if(psz == NULL) { dbgprintf("Custom_BindCDR: client part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("client ") - 1; /* skip "label" */ for( lenClient = 0 @@ -247,7 +244,7 @@ CODESTARTstrgen psz = (uchar*) strstr((char*) getMSG(pMsg), "view "); if(psz == NULL) { dbgprintf("Custom_BindCDR: view part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("view ") - 1; /* skip "label" */ for( lenView = 0 @@ -262,10 +259,10 @@ CODESTARTstrgen psz = (uchar*) strstr((char*) getMSG(pMsg), "query: "); if(psz == NULL) { dbgprintf("Custom_BindCDR: query part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("query: ") - 1; /* skip "label" */ - /* first find end-of-string to process */ + /* first find end-of-strihttp://www.rsyslog.com/doc/omruleset.htmlng to process */ while(*psz && (isdigit(*psz) || *psz == '.')) { psz++; } @@ -283,7 +280,7 @@ CODESTARTstrgen psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + ("); if(psz == NULL) { dbgprintf("Custom_BindCDR: ip part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("IN TXT + (") - 1; /* skip "label" */ for( lenIP = 0 @@ -302,7 +299,7 @@ CODESTARTstrgen */ if(isAllowed(szIP)) { DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } /* calculate len, constants for spaces and similar fixed strings */ @@ -317,8 +314,6 @@ CODESTARTstrgen memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1); iBuf = sizeof(SQL_STMT) - 1; - // SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); - memcpy(*ppBuf + iBuf, szDate, lenDate); iBuf += lenDate; ADD_SQL_DELIM -- cgit v1.2.3 From 12f126a811dd08553ac9387392c3a506eba3fe55 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 29 Mar 2011 11:59:22 +0200 Subject: manually merging ChangeLog --- ChangeLog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index d59871fe..fdd45aa1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,11 +1,11 @@ +--------------------------------------------------------------------------- +Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-?? - bugfix: rsyslog did not build with --disable-regexp configure option closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243 - bugfix: PRI was invalid on Solaris for message from local log socket - enhance: added $BOM system property to ease writing byte order masks - bugfix: RFC5424 parser confused by empty structured data closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237 ---------------------------------------------------------------------------- -Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-?? - bugfix: error return from strgen caused abort, now causes action to be ignored (just like a failed filter) - new sample plugin for a strgen to generate sql statement consumable -- cgit v1.2.3 From 9003632c7df5e6020b06b7d19e1c226ea46640de Mon Sep 17 00:00:00 2001 From: Christian Kastner Date: Tue, 29 Mar 2011 12:00:56 +0200 Subject: Force ANSI SQL treatment of strings for Postgres Signed-off-by: Rainer Gerhards --- plugins/ompgsql/ompgsql.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index ab8e4d2c..ea4b4b75 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -149,8 +149,13 @@ static rsRetVal initPgSQL(instanceData *pData, int bSilent) dbgprintf("host=%s dbname=%s uid=%s\n",pData->f_dbsrv,pData->f_dbname,pData->f_dbuid); + /* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may + * get all sorts of side effects (e.g.: backslash escapes) and warnings + */ + const char *PgConnectionOptions = "-c standard_conforming_strings=on"; + /* Connect to database */ - if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, NULL, NULL, + if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, PgConnectionOptions, NULL, pData->f_dbname, pData->f_dbuid, pData->f_dbpwd)) == NULL) { reportDBError(pData, bSilent); closePgSQL(pData); /* ignore any error we may get */ -- cgit v1.2.3 From c091e8491c0fa100dce951c09f8da77ee833059a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 29 Mar 2011 12:02:52 +0200 Subject: doc: adding bugfix to ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index fdd45aa1..90bc0f2d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-?? +- bugfix: ompgsql did not work properly with ANSI SQL strings + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229 - bugfix: rsyslog did not build with --disable-regexp configure option closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243 - bugfix: PRI was invalid on Solaris for message from local log socket -- cgit v1.2.3 From 0fb11a5a11e3dcb389d6139e24b0d8aa72164dd6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 29 Mar 2011 14:22:28 +0200 Subject: preparing for 5.7.10 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 90bc0f2d..e4c25889 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-?? +Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-29 - bugfix: ompgsql did not work properly with ANSI SQL strings closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229 - bugfix: rsyslog did not build with --disable-regexp configure option diff --git a/configure.ac b/configure.ac index 832a5b20..2e563490 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.9],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.10],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 807ac0da..f28d18d2 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.9 (beta branch) of rsyslog. +

    This documentation is for version 5.7.10 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 5bfa8d2a254e0ead0608b954bdab6dd2a7d9f3e0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 29 Mar 2011 14:31:14 +0200 Subject: testbench samples incorrect, corrected this was due to the old RFC5424 parser being considered correct, what it was not. As such, the samples generated from it were incorrect as well. Now with the corrected parsers, the tests failed. --- tests/testsuites/rfc5424-1.parse1 | 2 +- tests/testsuites/rfc5424-2.parse1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testsuites/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1 index 23836c9f..8a66d7ba 100644 --- a/tests/testsuites/rfc5424-1.parse1 +++ b/tests/testsuites/rfc5424-1.parse1 @@ -1,3 +1,3 @@ #Example from RFC5424, section 6.5 / sample 1 <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 -34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,BOM'su root' failed for lonvick on /dev/pts/8 diff --git a/tests/testsuites/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1 index a86fbc35..f5449e68 100644 --- a/tests/testsuites/rfc5424-2.parse1 +++ b/tests/testsuites/rfc5424-2.parse1 @@ -1,4 +1,4 @@ <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. -165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. +165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],%% It's time to make the do-nuts. #Example from RFC5424, section 6.5 / sample 2 #Only the first two lines are important, you may place anything behind them! -- cgit v1.2.3 From b59f8fd65d4f464ce3673439725db1606ec31329 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 30 Mar 2011 11:19:22 +0200 Subject: fixed problem in testbench & added new test (not yet integrated) The test is a setup scenario for this bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=241 --- plugins/imdiag/imdiag.c | 5 +- tests/diag.sh | 14 +++- tests/diagtalker.c | 6 +- tests/rcvr_fail_restore.sh | 96 ++++++++++++++++++++++++++ tests/testsuites/diag-common.conf | 5 +- tests/testsuites/rcvr_fail_restore_rcvr.conf | 8 +++ tests/testsuites/rcvr_fail_restore_sender.conf | 15 ++++ 7 files changed, 143 insertions(+), 6 deletions(-) create mode 100755 tests/rcvr_fail_restore.sh create mode 100644 tests/testsuites/rcvr_fail_restore_rcvr.conf create mode 100644 tests/testsuites/rcvr_fail_restore_sender.conf diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 404cebc7..ed6ef509 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -205,7 +205,7 @@ doInjectMsg(int iNum) DEFiRet; snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), - "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum); + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum); datetime.getCurrTime(&stTime, &ttGenTime); /* we now create our own message object and submit it to the queue */ @@ -247,6 +247,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); + DBGPRINTF("imdiag: %d messages injected\n", nMsgs); finalize_it: RETiRet; @@ -279,6 +280,7 @@ waitMainQEmpty(tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "mainqueue empty\n")); + DBGPRINTF("imdiag: mainqueue empty\n"); finalize_it: RETiRet; @@ -314,6 +316,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); + DBGPRINTF("imdiag: %d messages in main queue\n", iMsgQueueSize); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) { CHKiRet(waitMainQEmpty(pSess)); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) { diff --git a/tests/diag.sh b/tests/diag.sh index 8fbe42ed..d1242fb1 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -84,15 +84,27 @@ case $1 in exit 1 fi ;; + 'get-mainqueuesize') # show the current main queue size + if [ "$2" == "2" ] + then + echo getmainmsgqueuesize | ./diagtalker -p13501 + else + echo getmainmsgqueuesize | ./diagtalker + fi + ;; 'wait-queueempty') # wait for main message queue to be empty. $2 is the instance. if [ "$2" == "2" ] then - echo WaitMainQueueEmpty | ./diagtalker + echo WaitMainQueueEmpty | ./diagtalker -p13501 else echo WaitMainQueueEmpty | ./diagtalker fi ;; 'shutdown-when-empty') # shut rsyslogd down when main queue is empty. $2 is the instance. + if [ "$2" == "2" ] + then + echo Shutting down instance 2 + fi $srcdir/diag.sh wait-queueempty $2 kill `cat rsyslog$2.pid` # note: we do not wait for the actual termination! diff --git a/tests/diagtalker.c b/tests/diagtalker.c index 6a721e47..0d728bda 100644 --- a/tests/diagtalker.c +++ b/tests/diagtalker.c @@ -1,7 +1,7 @@ /* A yet very simple tool to talk to imdiag (this replaces the * previous Java implementation in order to get fewer dependencies). * - * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2010,2011 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -126,7 +126,7 @@ doProcessing() len = strlen(line); sendCmd(fd, line, len); waitRsp(fd, line, sizeof(line)); - printf("imdiag: %s", line); + printf("imdiag[%d]: %s", targetPort, line); } } @@ -139,7 +139,7 @@ int main(int argc, char *argv[]) int ret = 0; int opt; - while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) { + while((opt = getopt(argc, argv, "t:p:")) != -1) { switch (opt) { case 't': targetIP = optarg; break; diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh new file mode 100755 index 00000000..e2f0a0d8 --- /dev/null +++ b/tests/rcvr_fail_restore.sh @@ -0,0 +1,96 @@ +# Copyright (C) 2011 by Rainer Gerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[rcvr_fail_restore.sh\]: test failed receiver restore case +source $srcdir/diag.sh init +# +# STEP1: start both instances and send 1000 messages. +# Note: receiver is instance 2, sender instance 1. +# +# start up the instances. Note that the envrionment settings can be changed to +# set instance-specific debugging parameters! +export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="log2" +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +export RSYSLOG_DEBUGLOG="log" +#valgrind="valgrind" +source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf +# re-set params so that new instances do not thrash it... +unset RSYSLOG_DEBUG +unset RSYSLOG_DEBUGLOG + +# now inject the messages into instance 2. It will connect to instance 1, +# and that instance will record the data. +source $srcdir/diag.sh injectmsg 1 1000 +source $srcdir/diag.sh wait-queueempty +./msleep 3000 # make sure some retries happen (retry interval is set to 1 second) +# +# Step 2: shutdown receiver, then send some more data, which then +# needs to go into the queue. +# +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +source $srcdir/diag.sh injectmsg 1001 10000 +sleep 1 # we need to wait, otherwise we may be so fast that the receiver +# comes up before we have finally suspended the action +source $srcdir/diag.sh get-mainqueuesize +ls -l test-spool + +# +# Step 3: restart receiver, wait that the sender drains its queue +# +#export RSYSLOG_DEBUGLOG="log2" +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +echo waiting for sender to drain queue [may need a short while] +source $srcdir/diag.sh wait-queueempty +ls -l test-spool +OLDFILESIZE=$(stat -c%s test-spool/mainq.00000001) +echo file size to expect is $OLDFILESIZE + + +# +# Step 4: send new data. Queue files are not permitted to grow now +# (but one file continous to exist). +# +source $srcdir/diag.sh injectmsg 11001 10 +source $srcdir/diag.sh wait-queueempty + +# at this point, the queue file shall not have grown. Note +# that we MUST NOT shut down the instance right now, because it +# would clean up the queue files! So we need to do our checks +# first (here!). +ls -l test-spool +NEWFILESIZE=$(stat -c%s test-spool/mainq.00000001) +if [ $NEWFILESIZE != $OLDFILESIZE ] +then + echo file sizes do not match, expected $OLDFILESIZE, actual $NEWFILESIZE + echo this means that data has been written to the queue file where it + echo no longer should be written. + # abort will happen below, because we must ensure proper system shutdown + # HOWEVER, during actual testing it may be useful to do an exit here (so + # that e.g. the debug log is pointed right at the correct spot). + # exit 1 +fi + + +# +# Queue file size checks done. Now it is time to terminate the system +# and see if everything could be received (the usual check, done here +# for completeness, more or less as a bonus). +# +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown + +# now it is time to stop the receiver as well +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +# now abort test if we need to (due to filesize predicate) +if [ $NEWFILESIZE != $OLDFILESIZE ] +then + exit 1 +fi +# do the final check +source $srcdir/diag.sh seq-check 1 11010 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf index 9e9e28fe..d76e2d15 100644 --- a/tests/testsuites/diag-common.conf +++ b/tests/testsuites/diag-common.conf @@ -13,4 +13,7 @@ $IMDiagServerRun 13500 $template startupfile,"rsyslogd.started" # trick to use relative path names! :syslogtag, contains, "rsyslogd" ?startupfile -$ErrorMessagesToStderr off +# I have disabled the directive below, so that we see errors in testcase +# creation. I am not sure why it was present in the first place, so for +# now I just leave it commented out -- rgerhards, 2011-03-30 +#$ErrorMessagesToStderr off diff --git a/tests/testsuites/rcvr_fail_restore_rcvr.conf b/tests/testsuites/rcvr_fail_restore_rcvr.conf new file mode 100644 index 00000000..38ebad29 --- /dev/null +++ b/tests/testsuites/rcvr_fail_restore_rcvr.conf @@ -0,0 +1,8 @@ +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# then SENDER sends to this port (not tcpflood!) +$InputTCPServerRun 13515 + +$template outfmt,"%msg:F,58:2%\n" +:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/rcvr_fail_restore_sender.conf b/tests/testsuites/rcvr_fail_restore_sender.conf new file mode 100644 index 00000000..6b11aa4a --- /dev/null +++ b/tests/testsuites/rcvr_fail_restore_sender.conf @@ -0,0 +1,15 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +$WorkDirectory test-spool +$MainMsgQueueMaxFileSize 1g +$MainMsgQueueFileName mainq + +# we use the shortest resume interval a) to let the test not run too long +# and b) make sure some retries happen before the reconnect +$ActionResumeInterval 1 +$ActionResumeRetryCount -1 +*.* @@127.0.0.1:13515 -- cgit v1.2.3 From 00e1f24187ee814801e6969629b82a7ae030beaf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 30 Mar 2011 12:13:14 +0200 Subject: stop adding data to DA queue when low watermark has been reached potentially closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241 But needs more verification. --- runtime/queue.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/queue.c b/runtime/queue.c index ef6e843b..50ae307c 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1774,9 +1774,13 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) { DEFiRet; +//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis)); if(pThis->bEnqOnly) { iRet = RS_RET_TERMINATE_WHEN_IDLE; } + if(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk) { + iRet = RS_RET_TERMINATE_NOW; + } RETiRet; } -- cgit v1.2.3 From eccfbf71f9dbc235309c2ad40035d41620f5e5df Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 31 Mar 2011 15:14:22 +0200 Subject: improved testbench, updated ChangeLog to reflect recent bugfix After trying out some more things with the new code (after last bugfix), I now think that the bugfix is OK to use and without regressions. --- ChangeLog | 4 ++++ tests/Makefile.am | 6 ++++-- tests/rcvr_fail_restore.sh | 42 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index e4c25889..1eba3db9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,8 @@ --------------------------------------------------------------------------- +Version 5.8.0 [V5-stable] (rgerhards), 2011-04-?? +- bugfix: DA queue was never shutdown once it was started + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241 +--------------------------------------------------------------------------- Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-29 - bugfix: ompgsql did not work properly with ANSI SQL strings closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229 diff --git a/tests/Makefile.am b/tests/Makefile.am index ebbe0cf6..930aa304 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -56,8 +56,7 @@ TESTS += \ failover-rptd.sh \ failover-no-rptd.sh \ failover-no-basic.sh \ - queue-persist.sh - queue-persist.sh \ + rcvr_fail_restore.sh \ linkedlistqueue.sh endif @@ -289,6 +288,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ diag.sh \ testsuites/diag-common.conf \ testsuites/diag-common2.conf \ + rcvr_fail_restore.sh \ + testsuites/rcvr_fail_restore_rcvr.conf \ + testsuites/rcvr_fail_restore_sender.conf \ daqueue-persist.sh \ daqueue-persist-drvr.sh \ queue-persist.sh \ diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh index e2f0a0d8..79486f10 100755 --- a/tests/rcvr_fail_restore.sh +++ b/tests/rcvr_fail_restore.sh @@ -9,31 +9,32 @@ source $srcdir/diag.sh init # # start up the instances. Note that the envrionment settings can be changed to # set instance-specific debugging parameters! -export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUG="debug nostdout" #export RSYSLOG_DEBUGLOG="log2" source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 -export RSYSLOG_DEBUGLOG="log" +#export RSYSLOG_DEBUGLOG="log" #valgrind="valgrind" source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf # re-set params so that new instances do not thrash it... -unset RSYSLOG_DEBUG -unset RSYSLOG_DEBUGLOG +#unset RSYSLOG_DEBUG +#unset RSYSLOG_DEBUGLOG # now inject the messages into instance 2. It will connect to instance 1, # and that instance will record the data. source $srcdir/diag.sh injectmsg 1 1000 source $srcdir/diag.sh wait-queueempty -./msleep 3000 # make sure some retries happen (retry interval is set to 1 second) +./msleep 1000 # let things settle down a bit + # # Step 2: shutdown receiver, then send some more data, which then # needs to go into the queue. # + source $srcdir/diag.sh shutdown-when-empty 2 source $srcdir/diag.sh wait-shutdown 2 source $srcdir/diag.sh injectmsg 1001 10000 -sleep 1 # we need to wait, otherwise we may be so fast that the receiver -# comes up before we have finally suspended the action +./msleep 3000 # make sure some retries happen (retry interval is set to 3 second) source $srcdir/diag.sh get-mainqueuesize ls -l test-spool @@ -73,6 +74,31 @@ then # exit 1 fi +# +# We now do an extra test (so this is two in one ;)) to see if the DA +# queue can be reactivated after its initial shutdown. In essence, we +# redo steps 2 and 3. +# +# Step 5: stop receiver again, then send some more data, which then +# needs to go into the queue. +# +echo "*** done primary test *** now checking if DA can be restarted" +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +source $srcdir/diag.sh injectmsg 11011 10000 +sleep 1 # we need to wait, otherwise we may be so fast that the receiver +# comes up before we have finally suspended the action +source $srcdir/diag.sh get-mainqueuesize +ls -l test-spool + +# +# Step 6: restart receiver, wait that the sender drains its queue +# +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +echo waiting for sender to drain queue [may need a short while] +source $srcdir/diag.sh wait-queueempty +ls -l test-spool # # Queue file size checks done. Now it is time to terminate the system @@ -92,5 +118,5 @@ then exit 1 fi # do the final check -source $srcdir/diag.sh seq-check 1 11010 +source $srcdir/diag.sh seq-check 1 21010 source $srcdir/diag.sh exit -- cgit v1.2.3 From 400b9a27a5196971de9def21532b6dc1ef81306c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Apr 2011 16:36:38 +0200 Subject: doc bugfix: doc listed no-longer existing TLS limitations --- doc/rsyslog_tls.html | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/doc/rsyslog_tls.html b/doc/rsyslog_tls.html index bb312c77..286660d2 100644 --- a/doc/rsyslog_tls.html +++ b/doc/rsyslog_tls.html @@ -162,25 +162,11 @@ similar "smart" command on the client. It should show up in the respective server log file. If you dig out your sniffer, you should see that the traffic on the wire is actually protected.

    Limitations

    -

    The current implementation has a number of limitations. These -are -being worked on. Most importantly, neither the client nor the server -are authenticated. So while the message transfer is encrypted, you can -not be sure which peer you are talking to. Please note that this is a -limitation found in most real-world SSL syslog systems. Of course, that -is not an excuse for not yet providing this feature - but it tells you -that it is acceptable and can be worked around by proper firewalling, -ACLs and other organizational measures. Mutual authentication will be -added shortly to rsyslog.

    -

    Secondly, the plain tcp syslog listener -can currently listen to a single port, in a single mode. So if you use -a TLS-based listener, you can not run unencrypted syslog on the same -instance at the same time. A work-around is to run a second rsyslogd -instance. This limitation, too, is scheduled to be removed soon.

    The RELP transport can currently not be protected by TLS. A work-around is to use stunnel. TLS support for RELP will be added once plain TCP -syslog has sufficiently matured.

    +syslog has sufficiently matured and there either is some time left to do this +or we find a sponsor ;).

    Certificates

    In order to be really secure, certificates are needed. This is a short summary on how to generate the necessary certificates with -- cgit v1.2.3 From 770070a3caf22aee66b18bb98320a59d0ad41fa1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 8 Apr 2011 11:04:38 +0200 Subject: bugfix: race condition in deferred name resolution Note that this actually is a very small change, but I needed to shuffle a lot of code around in order to make it compile (due to required define order...). --- ChangeLog | 1 + runtime/msg.c | 232 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 118 insertions(+), 115 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1eba3db9..9d2085dc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 5.8.0 [V5-stable] (rgerhards), 2011-04-?? +- bugfix: race condition in deferred name resolution - bugfix: DA queue was never shutdown once it was started closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241 --------------------------------------------------------------------------- diff --git a/runtime/msg.c b/runtime/msg.c index e8e60963..d1e67aa2 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -287,6 +287,121 @@ static pthread_mutex_t mutTrimCtr; /* mutex to handle malloc trim */ static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); +/* The following functions will support advanced output module + * multithreading, once this is implemented. Currently, we + * include them as hooks only. The idea is that we need to guard + * some msg objects data fields against concurrent access if + * we run on multiple threads. Please note that in any case this + * is not necessary for calls from INPUT modules, because they + * construct the message object and do this serially. Only when + * the message is in the processing queue, multiple threads may + * access a single object. Consequently, there are no guard functions + * for "set" methods, as these are called during input. Only "get" + * functions that modify important structures have them. + * rgerhards, 2007-07-20 + * We now support locked and non-locked operations, depending on + * the configuration of rsyslog. To support this, we use function + * pointers. Initially, we start in non-locked mode. There, all + * locking operations call into dummy functions. When locking is + * enabled, the function pointers are changed to functions doing + * actual work. We also introduced another MsgPrepareEnqueue() function + * which initializes the locking structures, if needed. This is + * necessary because internal messages during config file startup + * processing are always created in non-locking mode. So we can + * not initialize locking structures during constructions. We now + * postpone this until when the message is fully constructed and + * enqueued. Then we know the status of locking. This has a nice + * side effect, and that is that during the initial creation of + * the Msg object no locking needs to be done, which results in better + * performance. -- rgerhards, 2008-01-05 + */ +static void (*funcLock)(msg_t *pMsg); +static void (*funcUnlock)(msg_t *pMsg); +static void (*funcDeleteMutex)(msg_t *pMsg); +void (*funcMsgPrepareEnqueue)(msg_t *pMsg); +#if 1 /* This is a debug aid */ +#define MsgLock(pMsg) funcLock(pMsg) +#define MsgUnlock(pMsg) funcUnlock(pMsg) +#else +#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; } +#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); } +#endif + +/* the next function is a dummy to be used by the looking functions + * when the class is not yet running in an environment where locking + * is necessary. Please note that the need to lock can (and will) change + * during a single run. Typically, this is depending on the operation mode + * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05 + */ +static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) +{ + /* empty be design */ +} + + +/* The following function prepares a message for enqueue into the queue. This is + * where a message may be accessed by multiple threads. This implementation here + * is the version for multiple concurrent acces. It initializes the locking + * structures. + * TODO: change to an iRet interface! -- rgerhards, 2008-07-14 + */ +static void MsgPrepareEnqueueLockingCase(msg_t *pThis) +{ + BEGINfunc + assert(pThis != NULL); + pthread_mutex_init(&pThis->mut, NULL); + pThis->bDoLock = 1; + ENDfunc +} + + +/* ... and now the locking and unlocking implementations: */ +static void MsgLockLockingCase(msg_t *pThis) +{ + /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ + assert(pThis != NULL); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_lock(&pThis->mut); +} + +static void MsgUnlockLockingCase(msg_t *pThis) +{ + /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ + assert(pThis != NULL); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_unlock(&pThis->mut); +} + +/* delete the mutex object on message destruction (locking case) + */ +static void MsgDeleteMutexLockingCase(msg_t *pThis) +{ + assert(pThis != NULL); + pthread_mutex_destroy(&pThis->mut); +} + +/* enable multiple concurrent access on the message object + * This works on a class-wide basis and can bot be undone. + * That is, if it is once enabled, it can not be disabled during + * the same run. When this function is called, no other thread + * must manipulate message objects. Then we would have race conditions, + * but guarding against this is counter-productive because it + * would cost additional time. Plus, it would be a programming error. + * rgerhards, 2008-01-05 + */ +rsRetVal MsgEnableThreadSafety(void) +{ + DEFiRet; + funcLock = MsgLockLockingCase; + funcUnlock = MsgUnlockLockingCase; + funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase; + funcDeleteMutex = MsgDeleteMutexLockingCase; + RETiRet; +} + +/* end locking functions */ + + static inline int getProtocolVersion(msg_t *pM) { return(pM->iProtocolVersion); @@ -306,6 +421,7 @@ resolveDNS(msg_t *pMsg) { uchar fromHostFQDN[NI_MAXHOST]; DEFiRet; + MsgLock(pMsg); CHKiRet(objUse(net, CORE_COMPONENT)); if(pMsg->msgFlags & NEEDS_DNSRESOL) { localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); @@ -315,6 +431,7 @@ resolveDNS(msg_t *pMsg) { } } finalize_it: + MsgUnlock(pMsg); if(iRet != RS_RET_OK) { /* best we can do: remove property */ MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); @@ -535,121 +652,6 @@ uchar *propIDToName(propid_t propID) } -/* The following functions will support advanced output module - * multithreading, once this is implemented. Currently, we - * include them as hooks only. The idea is that we need to guard - * some msg objects data fields against concurrent access if - * we run on multiple threads. Please note that in any case this - * is not necessary for calls from INPUT modules, because they - * construct the message object and do this serially. Only when - * the message is in the processing queue, multiple threads may - * access a single object. Consequently, there are no guard functions - * for "set" methods, as these are called during input. Only "get" - * functions that modify important structures have them. - * rgerhards, 2007-07-20 - * We now support locked and non-locked operations, depending on - * the configuration of rsyslog. To support this, we use function - * pointers. Initially, we start in non-locked mode. There, all - * locking operations call into dummy functions. When locking is - * enabled, the function pointers are changed to functions doing - * actual work. We also introduced another MsgPrepareEnqueue() function - * which initializes the locking structures, if needed. This is - * necessary because internal messages during config file startup - * processing are always created in non-locking mode. So we can - * not initialize locking structures during constructions. We now - * postpone this until when the message is fully constructed and - * enqueued. Then we know the status of locking. This has a nice - * side effect, and that is that during the initial creation of - * the Msg object no locking needs to be done, which results in better - * performance. -- rgerhards, 2008-01-05 - */ -static void (*funcLock)(msg_t *pMsg); -static void (*funcUnlock)(msg_t *pMsg); -static void (*funcDeleteMutex)(msg_t *pMsg); -void (*funcMsgPrepareEnqueue)(msg_t *pMsg); -#if 1 /* This is a debug aid */ -#define MsgLock(pMsg) funcLock(pMsg) -#define MsgUnlock(pMsg) funcUnlock(pMsg) -#else -#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; } -#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); } -#endif - -/* the next function is a dummy to be used by the looking functions - * when the class is not yet running in an environment where locking - * is necessary. Please note that the need to lock can (and will) change - * during a single run. Typically, this is depending on the operation mode - * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05 - */ -static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) -{ - /* empty be design */ -} - - -/* The following function prepares a message for enqueue into the queue. This is - * where a message may be accessed by multiple threads. This implementation here - * is the version for multiple concurrent acces. It initializes the locking - * structures. - * TODO: change to an iRet interface! -- rgerhards, 2008-07-14 - */ -static void MsgPrepareEnqueueLockingCase(msg_t *pThis) -{ - BEGINfunc - assert(pThis != NULL); - pthread_mutex_init(&pThis->mut, NULL); - pThis->bDoLock = 1; - ENDfunc -} - - -/* ... and now the locking and unlocking implementations: */ -static void MsgLockLockingCase(msg_t *pThis) -{ - /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ - assert(pThis != NULL); - if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ - pthread_mutex_lock(&pThis->mut); -} - -static void MsgUnlockLockingCase(msg_t *pThis) -{ - /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ - assert(pThis != NULL); - if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ - pthread_mutex_unlock(&pThis->mut); -} - -/* delete the mutex object on message destruction (locking case) - */ -static void MsgDeleteMutexLockingCase(msg_t *pThis) -{ - assert(pThis != NULL); - pthread_mutex_destroy(&pThis->mut); -} - -/* enable multiple concurrent access on the message object - * This works on a class-wide basis and can bot be undone. - * That is, if it is once enabled, it can not be disabled during - * the same run. When this function is called, no other thread - * must manipulate message objects. Then we would have race conditions, - * but guarding against this is counter-productive because it - * would cost additional time. Plus, it would be a programming error. - * rgerhards, 2008-01-05 - */ -rsRetVal MsgEnableThreadSafety(void) -{ - DEFiRet; - funcLock = MsgLockLockingCase; - funcUnlock = MsgUnlockLockingCase; - funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase; - funcDeleteMutex = MsgDeleteMutexLockingCase; - RETiRet; -} - -/* end locking functions */ - - /* This is common code for all Constructors. It is defined in an * inline'able function so that we can save a function call in the * actual constructors (otherwise, the msgConstruct would need -- cgit v1.2.3 From 0a6e7fcb2201c097da717b54ec82bdd98c4a5452 Mon Sep 17 00:00:00 2001 From: Marius Tomaschewski Date: Mon, 4 Apr 2011 17:29:28 +0200 Subject: corrected ActionSendResendLastMsgOnReconnect docs Signed-off-by: Marius Tomaschewski --- doc/rsyslog_conf_global.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 7d121a4a..a5d69f1d 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -93,7 +93,7 @@ default 60000 (1 minute)]

  • $ActionQueueWorkerThreadMinumumMessages <number>, default 100
  • $ActionResumeInterval
  • $ActionResumeRetryCount <number> [default 0, -1 means eternal]
    • -
  • $ActionSendResendLastMsgOnReconn <[on/off]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. +
  • $ActionSendResendLastMsgOnReconnect <[on/off]> specifies if the last message is to be resend when a connecition breaks and has been reconnected. May increase reliability, but comes at the risk of message duplication.
  • $ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action
  • $ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver (driver-specific)
  • $ActionSendStreamDriverAuthMode <mode>,  authentication mode to use with the stream driver. Note that this directive requires TLS -- cgit v1.2.3 From 096049778ea86ccf7f05b75f8e5b378207e0b3f7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 Apr 2011 10:29:25 +0200 Subject: confirmed that the race condition fix solved bug #238 --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 9d2085dc..e6237b41 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,9 @@ --------------------------------------------------------------------------- Version 5.8.0 [V5-stable] (rgerhards), 2011-04-?? - bugfix: race condition in deferred name resolution + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=238 + Special thanks to Marcin for his persistence in helping to solve this + bug. - bugfix: DA queue was never shutdown once it was started closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241 --------------------------------------------------------------------------- -- cgit v1.2.3 From 9348c80744b29fb5f91b5d8edd3f9070f0d0347b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 Apr 2011 10:35:00 +0200 Subject: cleanup: initially thought temporary debug message made durable ;) It has proven to be very valuable, so now it is properly formatted inside the source. --- runtime/ruleset.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/ruleset.c b/runtime/ruleset.c index c9c64a38..5ee2a55a 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -223,7 +223,7 @@ processBatch(batch_t *pBatch) DEFiRet; assert(pBatch != NULL); -dbgprintf("ZZZ: processBatch: batch of %d elements must be processed\n", pBatch->nElem); + DBGPRINTF("processBatch: batch of %d elements must be processed\n", pBatch->nElem); if(pBatch->bSingleRuleset) { pThis = batchGetRuleset(pBatch); if(pThis == NULL) @@ -235,7 +235,7 @@ dbgprintf("ZZZ: processBatch: batch of %d elements must be processed\n", pBatch- } finalize_it: -dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet); + DBGPRINTF("ruleset.ProcessMsg() returns %d\n", iRet); RETiRet; } -- cgit v1.2.3