From 3cbd7300c47991aadca00db1b028fd3d5d551040 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Sep 2010 12:24:22 +0200 Subject: added module impstat to emit periodic statistics on rsyslog counters This is a *very first* and *very rough* and *very featureless* first shot at this functionality. It is assumed that we will enhance the stats system as a by-line while doing other development. --- ChangeLog | 6 +- Makefile.am | 5 + configure.ac | 15 +++ doc/Makefile.am | 1 + doc/impstats.html | 60 ++++++++++++ doc/rsyslog_conf_modules.html | 1 + plugins/impstats/Makefile.am | 6 ++ plugins/impstats/impstats.c | 211 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 doc/impstats.html create mode 100644 plugins/impstats/Makefile.am create mode 100644 plugins/impstats/impstats.c diff --git a/ChangeLog b/ChangeLog index cfd1831d..0f8b625d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,10 @@ --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-?? +- added module impstat to emit periodic statistics on rsyslog counters - support for systemd officially added -- acquire /dev/log socket optionally from systemd - thanks to Lennart Poettering for this patch + * acquire /dev/log socket optionally from systemd + thanks to Lennart Poettering for this patch + * sd-systemd API added as part of rsyslog runtime library --------------------------------------------------------------------------- Version 5.5.7 [V5-BETA] (rgerhards), 2010-08-09 - changed omudpspoof default spoof address to simplify typical use case diff --git a/Makefile.am b/Makefile.am index 76a38322..bc6d8dd0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -59,6 +59,10 @@ if ENABLE_IMKLOG SUBDIRS += plugins/imklog endif +if ENABLE_IMPSTATS +SUBDIRS += plugins/impstats +endif + if ENABLE_IMSOLARIS SUBDIRS += plugins/imsolaris endif @@ -191,5 +195,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-omuxsock \ --enable-shave \ --enable-extended-tests \ + --enable-impstats \ --enable-memcheck ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 9c005a19..c52749c8 100644 --- a/configure.ac +++ b/configure.ac @@ -792,6 +792,19 @@ AC_ARG_ENABLE(imptcp, AM_CONDITIONAL(ENABLE_IMPTCP, test x$enable_imptcp = xyes) +# settings for the pstats input module +AC_ARG_ENABLE(impstats, + [AS_HELP_STRING([--enable-impstats],[periodic statistics module enabled @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_impstats="yes" ;; + no) enable_impstats="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-impstats) ;; + esac], + [enable_impstats=no] +) +AM_CONDITIONAL(ENABLE_IMPSTATS, test x$enable_impstats = xyes) + + # settings for the omprog output module AC_ARG_ENABLE(omprog, [AS_HELP_STRING([--enable-omprog],[Compiles omprog module @<:@default=no@:>@])], @@ -1013,6 +1026,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imfile/Makefile \ plugins/imsolaris/Makefile \ plugins/imptcp/Makefile \ + plugins/impstats/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ plugins/omtesting/Makefile \ @@ -1049,6 +1063,7 @@ echo " plain tcp input module enabled: $enable_imptcp" echo " imdiag enabled: $enable_imdiag" echo " file input module enabled: $enable_imfile" echo " Solaris input module enabled: $enable_imsolaris" +echo " periodic statistics module enabled: $enable_impstats" echo " input template module will be compiled: $enable_imtemplate" echo echo "---{ output plugins }---" diff --git a/doc/Makefile.am b/doc/Makefile.am index d4df740a..a5393cbe 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -41,6 +41,7 @@ html_files = \ imfile.html \ imtcp.html \ imptcp.html \ + impstats.html \ imgssapi.html \ imrelp.html \ imsolaris.html \ diff --git a/doc/impstats.html b/doc/impstats.html new file mode 100644 index 00000000..0f4096ab --- /dev/null +++ b/doc/impstats.html @@ -0,0 +1,60 @@ + + +Periodic Statistics of Internal Counters (impstats) + + +back + +

Input Module to Generate Periodic Statistics of Internal Counters

+

Module Name:    impstats

+

Available since: 5.7.0+, 6.1.1+ +

Author: Rainer Gerhards <rgerhards@adiscon.com>

+

Description:

+

This module provides periodic output of rsyslog internal counters. +Note that the whole statistics system is currently under development. So +availabilty and format of counters may change and is not yet stable (so be +prepared to change your trending scripts when you upgrade to a newer rsyslog version). +

The set of available counters will be output as a set of syslog messages. This +output is periodic, with the interval being configurable (default is 5 minutes). +Be sure that your configuration records the counter messages (default is syslog.info). +

Note that loading this module has impact on rsyslog performance. Depending on +settings, this impact may be severe (for high-load environments). +

+

Configuration Directives:

+ +Caveats/Known Bugs: + +

Sample:

+

This activates the module and records messages to /var/log/rsyslog-stats in 10 minute intervals:
+

+ +

[rsyslog.conf overview] +[manual index] [rsyslog site]

+

This documentation is part of the +rsyslog +project.
+Copyright © 2010 by Rainer +Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

+ diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 2a64461d..85899954 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -44,6 +44,7 @@ to message generators.
  • imuxsock - unix sockets, including the system log socket
  • imsolaris - input for the Sun Solaris system log source
  • im3195 - accepts syslog messages via RFC 3195
    • +
  • impstats - provides periodic statistics of rsyslog internal counters

    • Output Modules

    diff --git a/plugins/impstats/Makefile.am b/plugins/impstats/Makefile.am new file mode 100644 index 00000000..187bbca4 --- /dev/null +++ b/plugins/impstats/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = impstats.la + +impstats_la_SOURCES = impstats.c +impstats_la_CPPFLAGS = $(RSRT_CFLAGS) -I$(top_srcdir) $(PTHREADS_CFLAGS) +impstats_la_LDFLAGS = -module -avoid-version +impstats_la_LIBADD = diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c new file mode 100644 index 00000000..5f0d19eb --- /dev/null +++ b/plugins/impstats/impstats.c @@ -0,0 +1,211 @@ +/* impstats.c + * A module to periodically output statistics gathered by rsyslog. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c) + * This file is under development and has not yet arrived at being fully + * self-contained and a real object. So far, it is mostly an excerpt + * of the "old" message code without any modifications. However, it + * helps to have things at the right place one we go to the meat of it. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "cfsysline.h" +#include "module-template.h" +#include "errmsg.h" +#include "msg.h" +#include "srUtils.h" +#include "unicode-helper.h" +#include "glbl.h" +#include "prop.h" + +MODULE_TYPE_INPUT + +/* defines */ +#define DEFAULT_STATS_PERIOD (5 * 60) +#define DEFAULT_FACILITY 5 /* syslog */ +#define DEFAULT_SEVERITY 6 /* info */ + +/* Module static data */ +DEF_IMOD_STATIC_DATA +DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) +typedef struct configSettings_s { + int iStatsInterval; + int iFacility; + int iSeverity; +} configSettings_t; + +static configSettings_t cs; + +static prop_t *pInputName = NULL; +static prop_t *pLocalHostIP = NULL; + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +static inline void +initConfigSettings(void) +{ + cs.iStatsInterval = DEFAULT_STATS_PERIOD; + cs.iFacility = DEFAULT_FACILITY; + cs.iSeverity = DEFAULT_SEVERITY; +} + + +/* actually submit a message to the rsyslog core + */ +static inline rsRetVal +doSubmitMsg(uchar *line) +{ + msg_t *pMsg; + DEFiRet; + + CHKiRet(msgConstruct(&pMsg)); + MsgSetInputName(pMsg, pInputName); + MsgSetRawMsgWOSize(pMsg, (char*)line); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); + MsgSetRcvFromIP(pMsg, pLocalHostIP); + MsgSetMSGoffs(pMsg, 0); + MsgSetTAG(pMsg, UCHAR_CONSTANT("rsyslogd-pstats:"), sizeof("rsyslogd-pstats:") - 1); + pMsg->iFacility = cs.iFacility; + pMsg->iSeverity = cs.iSeverity; + pMsg->msgFlags = 0; + + submitMsg(pMsg); + +finalize_it: + RETiRet; + +} + + +/* the function to generate the actual statistics messages + * rgerhards, 2010-09-09 + */ +static inline void +generateStatsMsgs(void) +{ + int iMsgQueueSize; + uchar msg[1024]; + rsRetVal iRet; + + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + snprintf((char*)msg, sizeof(msg), "mainqueue size=%d", iMsgQueueSize); + doSubmitMsg(msg); + +finalize_it: + /*empty statement needed per C syntax*/; +} + + +BEGINrunInput +CODESTARTrunInput + /* this is an endless loop - it is terminated when the thread is + * signalled to do so. This, however, is handled by the framework, + * right into the sleep below. + */ + while(1) { +dbgprintf("impstats: stats interval %d seconds - going to sleep\n", cs.iStatsInterval); + srSleep(cs.iStatsInterval, 0); /* seconds, micro seconds */ +dbgprintf("impstats: woke up\n"); + + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + + generateStatsMsgs(); + } +ENDrunInput + + +BEGINwillRun +CODESTARTwillRun + DBGPRINTF("impstats: stats interval %d seconds\n", cs.iStatsInterval); + if(cs.iStatsInterval == 0) + ABORT_FINALIZE(RS_RET_NO_RUN); +finalize_it: +ENDwillRun + + +BEGINafterRun +CODESTARTafterRun +ENDafterRun + + +BEGINmodExit +CODESTARTmodExit + /* release objects we used */ + objRelease(glbl, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); +ENDmodExit + + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + initConfigSettings(); + return RS_RET_OK; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + DBGPRINTF("impstats version %s loading\n", VERSION); + initConfigSettings(); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatsinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatfacility", 0, eCmdHdlrInt, NULL, &cs.iFacility, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatseverity", 0, eCmdHdlrInt, NULL, &cs.iSeverity, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("rsyslogd-pstats"), sizeof("rsyslogd-pstats") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + + CHKiRet(prop.Construct(&pLocalHostIP)); + CHKiRet(prop.SetString(pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pLocalHostIP)); +ENDmodInit +/* vi:set ai: + */ -- cgit v1.2.3 From 3d50fd6a67b7d0bf31628aea14128a47dd5d326f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Sep 2010 12:37:11 +0200 Subject: fixed small memory leak, corrected input name --- plugins/impstats/impstats.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index 5f0d19eb..a818d20c 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -166,6 +166,8 @@ ENDafterRun BEGINmodExit CODESTARTmodExit + prop.Destruct(&pInputName); + prop.Destruct(&pLocalHostIP); /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); @@ -200,7 +202,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(prop.Construct(&pInputName)); - CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("rsyslogd-pstats"), sizeof("rsyslogd-pstats") - 1)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("impstats"), sizeof("impstats") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); CHKiRet(prop.Construct(&pLocalHostIP)); -- cgit v1.2.3 From e86cb62f1299ef18732f7b3b87d45a840ee38f1e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 13 Sep 2010 15:43:56 +0200 Subject: improved statistics-gathering subsystem ... well, actually this is a first real implementation of this subsystem. I have added a counter registry, a way to access the countres (as readable string) and a way to define and maintem them. Also, module impstats has been updated to utilize the new system. Finally, I added some counters. I hope that this sets the baseline for useful future enhancements. --- doc/impstats.html | 2 + plugins/impstats/impstats.c | 39 ++++-- plugins/imuxsock/imuxsock.c | 18 +++ runtime/Makefile.am | 6 +- runtime/queue.c | 32 +++++ runtime/queue.h | 6 + runtime/rsyslog.c | 3 + runtime/rsyslog.h | 2 + runtime/statsobj.c | 315 ++++++++++++++++++++++++++++++++++++++++++++ runtime/statsobj.h | 124 +++++++++++++++++ 10 files changed, 533 insertions(+), 14 deletions(-) create mode 100644 runtime/statsobj.c create mode 100644 runtime/statsobj.h diff --git a/doc/impstats.html b/doc/impstats.html index 0f4096ab..3b4191e8 100644 --- a/doc/impstats.html +++ b/doc/impstats.html @@ -37,6 +37,8 @@ is 6 (info).This is useful for filtering messages. Caveats/Known Bugs:
      +
    • This module MUST be loaded right at the top of rsyslog.conf, otherwise +stats may not get turned on in all places.
    • experimental code

    Sample:

    diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index a818d20c..1312a4e8 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -45,6 +45,7 @@ #include "srUtils.h" #include "unicode-helper.h" #include "glbl.h" +#include "statsobj.h" #include "prop.h" MODULE_TYPE_INPUT @@ -58,6 +59,9 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(statsobj) +DEFobjCurrIf(errmsg) + typedef struct configSettings_s { int iStatsInterval; int iFacility; @@ -112,22 +116,25 @@ finalize_it: } +/* callback for statsobj + * Note: usrptr exists only to satisfy requirements of statsobj callback interface! + */ +static rsRetVal +doStatsLine(void __attribute__((unused)) *usrptr, cstr_t *cstr) +{ + DEFiRet; + doSubmitMsg(rsCStrGetSzStrNoNULL(cstr)); + RETiRet; +} + + /* the function to generate the actual statistics messages * rgerhards, 2010-09-09 */ static inline void generateStatsMsgs(void) { - int iMsgQueueSize; - uchar msg[1024]; - rsRetVal iRet; - - CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); - snprintf((char*)msg, sizeof(msg), "mainqueue size=%d", iMsgQueueSize); - doSubmitMsg(msg); - -finalize_it: - /*empty statement needed per C syntax*/; + statsobj.GetAllStatsLines(doStatsLine, NULL); } @@ -138,9 +145,7 @@ CODESTARTrunInput * right into the sleep below. */ while(1) { -dbgprintf("impstats: stats interval %d seconds - going to sleep\n", cs.iStatsInterval); srSleep(cs.iStatsInterval, 0); /* seconds, micro seconds */ -dbgprintf("impstats: woke up\n"); if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ @@ -151,10 +156,16 @@ ENDrunInput BEGINwillRun + rsRetVal localRet; CODESTARTwillRun DBGPRINTF("impstats: stats interval %d seconds\n", cs.iStatsInterval); if(cs.iStatsInterval == 0) ABORT_FINALIZE(RS_RET_NO_RUN); + localRet = statsobj.EnableStats(); + if(localRet != RS_RET_OK) { + errmsg.LogError(0, localRet, "impstat: error enabling statistics gathering"); + ABORT_FINALIZE(RS_RET_NO_RUN); + } finalize_it: ENDwillRun @@ -171,6 +182,8 @@ CODESTARTmodExit /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); ENDmodExit @@ -196,6 +209,8 @@ CODEmodInit_QueryRegCFSLineHdlr initConfigSettings(); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatsinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatfacility", 0, eCmdHdlrInt, NULL, &cs.iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatseverity", 0, eCmdHdlrInt, NULL, &cs.iSeverity, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index db53fcb6..d976c676 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -48,6 +48,7 @@ #include "debug.h" #include "unlimited_select.h" #include "sd-daemon.h" +#include "statsobj.h" MODULE_TYPE_INPUT @@ -72,6 +73,10 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(statsobj) + +statsobj_t *modStats; +STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ @@ -269,6 +274,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); + STATSCOUNTER_INC(ctrSubmit, mutCtrSubmit); finalize_it: RETiRet; } @@ -453,9 +459,12 @@ ENDafterRun BEGINmodExit CODESTARTmodExit + statsobj.Destruct(&modStats); + objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); ENDmodExit @@ -502,6 +511,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); @@ -547,6 +557,14 @@ CODEmodInit_QueryRegCFSLineHdlr setSystemLogTimestampIgnore, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&modStats)); + CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), + ctrType_IntCtr, &ctrSubmit)); + CHKiRet(statsobj.ConstructFinalize(modStats)); + ENDmodInit /* vim:set ai: */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 5b2598b2..01b224e7 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -45,6 +45,8 @@ librsyslog_la_SOURCES = \ modules.h \ apc.c \ apc.h \ + statsobj.c \ + statsobj.h \ sync.c \ sync.h \ expr.c \ @@ -81,8 +83,8 @@ librsyslog_la_SOURCES = \ prop.h \ cfsysline.c \ cfsysline.h \ - sd-daemon.c \ - sd-daemon.h \ + sd-daemon.c \ + sd-daemon.h \ \ \ ../action.h \ diff --git a/runtime/queue.c b/runtime/queue.c index 60d17086..387c15c2 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -58,6 +58,7 @@ #include "errmsg.h" #include "datetime.h" #include "unicode-helper.h" +#include "statsobj.h" #include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS @@ -70,6 +71,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(strm) DEFobjCurrIf(errmsg) DEFobjCurrIf(datetime) +DEFobjCurrIf(statsobj) /* forward-definitions */ static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); @@ -1817,6 +1819,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; uchar pszBuf[64]; + uchar *qName; size_t lenBuf; ASSERT(pThis != NULL); @@ -1886,6 +1889,27 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ qqueueAdviseMaxWorkers(pThis); pThis->bQueueStarted = 1; + /* support statistics gathering */ + qName = obj.GetName((obj_t*)pThis); + CHKiRet(statsobj.Construct(&pThis->statsobj)); + CHKiRet(statsobj.SetName(pThis->statsobj, qName)); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("size"), + ctrType_Int, &pThis->iQueueSize)); + + STATSCOUNTER_INIT(pThis->ctrEnqueued, pThis->mutCtrEnqueued); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("enqueued"), + ctrType_IntCtr, &pThis->ctrEnqueued)); + + STATSCOUNTER_INIT(pThis->ctrFull, pThis->mutCtrFull); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("full"), + ctrType_IntCtr, &pThis->ctrFull)); + + pThis->ctrMaxqsize = 0; + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("maxqsize"), + ctrType_Int, &pThis->ctrMaxqsize)); + + CHKiRet(statsobj.ConstructFinalize(pThis->statsobj)); + finalize_it: RETiRet; } @@ -2119,6 +2143,10 @@ CODESTARTobjDestruct(qqueue) free(pThis->pszFilePrefix); free(pThis->pszSpoolDir); + + /* some queues do not provide stats and thus have no statsobj! */ + if(pThis->statsobj != NULL) + statsobj.Destruct(&pThis->statsobj); ENDobjDestruct(qqueue) @@ -2178,6 +2206,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) DEFiRet; struct timespec t; + STATSCOUNTER_INC(pThis->ctrEnqueued, pThis->mutCtrEnqueued); /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); @@ -2225,6 +2254,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); timeoutComp(&t, pThis->toEnq); + STATSCOUNTER_INC(pThis->ctrFull, pThis->mutCtrFull); // TODO : handle enqOnly => discard! if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); @@ -2235,6 +2265,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); + STATSCOUNTER_SETMAX_NOMUT(pThis->ctrMaxqsize, pThis->iQueueSize); finalize_it: RETiRet; @@ -2414,6 +2445,7 @@ BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/queue.h b/runtime/queue.h index 1c758134..38e248cd 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -29,6 +29,7 @@ #include "wtp.h" #include "batch.h" #include "stream.h" +#include "statsobj.h" /* support for the toDelete list */ typedef struct toDeleteLst_s toDeleteLst_t; @@ -165,6 +166,11 @@ struct queue_s { } tVars; DEF_ATOMIC_HELPER_MUT(mutQueueSize); DEF_ATOMIC_HELPER_MUT(mutLogDeq); + /* for statistics subsystem */ + statsobj_t *statsobj; + STATSCOUNTER_DEF(ctrEnqueued, mutCtrEnqueued); + STATSCOUNTER_DEF(ctrFull, mutCtrFull); + int ctrMaxqsize; }; diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index a9794840..8baa2b59 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -82,6 +82,7 @@ #include "ruleset.h" #include "parser.h" #include "strgen.h" +#include "statsobj.h" #include "atomic.h" /* forward definitions */ @@ -150,6 +151,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) * class immediately after it is initialized. And, of course, we load those classes * first that we use ourselfs... -- rgerhards, 2008-03-07 */ + if(ppErrObj != NULL) *ppErrObj = "statsobj"; + CHKiRet(statsobjClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "prop"; CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 34eaedca..7b2655b0 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -150,6 +150,8 @@ typedef struct parser_s parser_t; typedef struct parserList_s parserList_t; typedef struct strgen_s strgen_t; typedef struct strgenList_s strgenList_t; +typedef struct statsobj_s statsobj_t; +typedef struct statsctr_s statsctr_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ diff --git a/runtime/statsobj.c b/runtime/statsobj.c new file mode 100644 index 00000000..e1a89cf4 --- /dev/null +++ b/runtime/statsobj.c @@ -0,0 +1,315 @@ +/* The statsobj object. + * + * This object provides a statistics-gathering facility inside rsyslog. This + * functionality will be pragmatically implemented and extended. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include + +#include "rsyslog.h" +#include "unicode-helper.h" +#include "obj.h" +#include "statsobj.h" +#include "sysvar.h" +#include "srUtils.h" +#include "stringbuf.h" + + +/* externally-visiable data (see statsobj.h for explanation) */ +int GatherStats = 0; + +/* static data */ +DEFobjStaticHelpers + +/* doubly linked list of stats objects. Object is automatically linked to it + * upon construction. Enqueue always happens at the front (simplifies logic). + */ +static statsobj_t *objRoot = NULL; +static statsobj_t *objLast = NULL; + +static pthread_mutex_t mutStats; + +/* ------------------------------ statsobj linked list maintenance ------------------------------ */ + +static inline void +addToObjList(statsobj_t *pThis) +{ + pthread_mutex_lock(&mutStats); + pThis->prev = objLast; + if(objLast != NULL) + objLast->next = pThis; + objLast = pThis; + if(objRoot == NULL) + objRoot = pThis; + pthread_mutex_unlock(&mutStats); +} + + +static inline void +removeFromObjList(statsobj_t *pThis) +{ + pthread_mutex_lock(&mutStats); + if(pThis->prev != NULL) + pThis->prev->next = pThis->next; + if(pThis->next != NULL) + pThis->next->prev = pThis->prev; + if(objLast == pThis) + objLast = pThis->prev; + if(objRoot == pThis) + objRoot = pThis->next; + pthread_mutex_unlock(&mutStats); +} + + +static inline void +addCtrToList(statsobj_t *pThis, ctr_t *pCtr) +{ + pthread_mutex_lock(&pThis->mutCtr); + pCtr->prev = pThis->ctrLast; + if(pThis->ctrLast != NULL) + pThis->ctrLast->next = pCtr; + pThis->ctrLast = pCtr; + if(pThis->ctrRoot == NULL) + pThis->ctrRoot = pCtr; + pthread_mutex_unlock(&pThis->mutCtr); +} + +/* ------------------------------ methods ------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(statsobj) /* be sure to specify the object type also in END macro! */ + pthread_mutex_init(&pThis->mutCtr, NULL); + pThis->ctrLast = NULL; + pThis->ctrRoot = NULL; +ENDobjConstruct(statsobj) + + +/* ConstructionFinalizer + */ +static rsRetVal +statsobjConstructFinalize(statsobj_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, statsobj); + addToObjList(pThis); + RETiRet; +} + + +/* set name. Note that we make our own copy of the memory, caller is + * responsible to free up name it passes in (if required). + */ +static rsRetVal +setName(statsobj_t *pThis, uchar *name) +{ + DEFiRet; + CHKmalloc(pThis->name = ustrdup(name)); +finalize_it: + RETiRet; +} + + +/* add a counter to an object + * ctrName is duplicated, caller must free it if requried + */ +static rsRetVal +addCounter(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr) +{ + ctr_t *ctr; + DEFiRet; + + CHKmalloc(ctr = malloc(sizeof(ctr_t))); + ctr->next = NULL; + ctr->prev = NULL; + CHKmalloc(ctr->name = ustrdup(ctrName)); + ctr->ctrType = ctrType; + switch(ctrType) { + case ctrType_IntCtr: + ctr->val.pIntCtr = (intctr_t*) pCtr; + break; + case ctrType_Int: + ctr->val.pInt = (int*) pCtr; + break; + } + addCtrToList(pThis, ctr); + +finalize_it: + RETiRet; +} + + +/* get all the object's countes together with object name as one line. + */ +static rsRetVal +getStatsLine(statsobj_t *pThis, cstr_t **ppcstr) +{ + cstr_t *pcstr; + ctr_t *pCtr; + DEFiRet; + + CHKiRet(cstrConstruct(&pcstr)); + rsCStrAppendStr(pcstr, pThis->name); + rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT(": "), 2); + + /* now add all counters to this line */ + pthread_mutex_lock(&pThis->mutCtr); + for(pCtr = pThis->ctrRoot ; pCtr != NULL ; pCtr = pCtr->next) { + rsCStrAppendStr(pcstr, pCtr->name); + cstrAppendChar(pcstr, '='); + switch(pCtr->ctrType) { + case ctrType_IntCtr: + rsCStrAppendInt(pcstr, *(pCtr->val.pIntCtr)); // TODO: OK????? + break; + case ctrType_Int: + rsCStrAppendInt(pcstr, *(pCtr->val.pInt)); + break; + } + cstrAppendChar(pcstr, ' '); + } + pthread_mutex_unlock(&pThis->mutCtr); + + CHKiRet(cstrFinalize(pcstr)); + *ppcstr = pcstr; + +finalize_it: + RETiRet; +} + + +/* this function can be used to obtain all stats lines. In this case, + * a callback must be provided. This module than iterates over all objects and + * submits each stats line to the callback. The callback has two parameters: + * the first one is a caller-provided void*, the second one the cstr_t with the + * line. If the callback reports an error, processing is stopped. + */ +static rsRetVal +getAllStatsLines(rsRetVal(*cb)(void*, cstr_t*), void *usrptr) +{ + statsobj_t *o; + cstr_t *cstr; + DEFiRet; + + for(o = objRoot ; o != NULL ; o = o->next) { + CHKiRet(getStatsLine(o, &cstr)); + CHKiRet(cb(usrptr, cstr)); + rsCStrDestruct(&cstr); + } + +finalize_it: + RETiRet; +} + + +/* Enable statistics gathering. currently there is no function to disable it + * again, as this is right now not needed. + */ +static rsRetVal +enableStats() +{ + GatherStats = 1; + return RS_RET_OK; +} + + +/* destructor for the statsobj object */ +BEGINobjDestruct(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ + ctr_t *ctr, *ctrToDel; +CODESTARTobjDestruct(statsobj) + removeFromObjList(pThis); + + /* destruct counters */ + ctr = pThis->ctrRoot; + while(ctr != NULL) { + ctrToDel = ctr; + ctr = ctr->next; + free(ctrToDel->name); + free(ctrToDel); + } + + pthread_mutex_destroy(&pThis->mutCtr); + free(pThis->name); +ENDobjDestruct(statsobj) + + +/* debugprint for the statsobj object */ +BEGINobjDebugPrint(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(statsobj) + dbgoprint((obj_t*) pThis, "statsobj object, currently no state info available\n"); +ENDobjDebugPrint(statsobj) + + +/* queryInterface function + */ +BEGINobjQueryInterface(statsobj) +CODESTARTobjQueryInterface(statsobj) + if(pIf->ifVersion != statsobjCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = statsobjConstruct; + pIf->ConstructFinalize = statsobjConstructFinalize; + pIf->Destruct = statsobjDestruct; + pIf->DebugPrint = statsobjDebugPrint; + pIf->SetName = setName; + pIf->GetStatsLine = getStatsLine; + pIf->GetAllStatsLines = getAllStatsLines; + pIf->AddCounter = addCounter; + pIf->EnableStats = enableStats; +finalize_it: +ENDobjQueryInterface(statsobj) + + +/* Initialize the statsobj class. Must be called as the very first method + * before anything else is called inside this class. + */ +BEGINAbstractObjClassInit(statsobj, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, statsobjDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, statsobjConstructFinalize); + + /* init other data items */ + pthread_mutex_init(&mutStats, NULL); + +ENDObjClassInit(statsobj) + +/* Exit the class. + */ +BEGINObjClassExit(statsobj, OBJ_IS_CORE_MODULE) /* class, version */ + /* release objects we no longer need */ + pthread_mutex_destroy(&mutStats); +ENDObjClassExit(statsobj) diff --git a/runtime/statsobj.h b/runtime/statsobj.h new file mode 100644 index 00000000..1ab02182 --- /dev/null +++ b/runtime/statsobj.h @@ -0,0 +1,124 @@ +/* The statsobj object. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STATSOBJ_H +#define INCLUDED_STATSOBJ_H + +#include "atomic.h" + +/* The following data item is somewhat dirty, in that it does not follow + * our usual object calling conventions. However, much like with "Debug", we + * do this to gain speed. If we finally come to a platform that does not + * provide resolution of names for dynamically loaded modules, we need to find + * a work-around, but until then, we use the direct access. + * If set to 0, statistics are not gathered, otherwise they are. + */ +extern int GatherStats; + +/* our basic counter type -- need 32 bit on 32 bit platform. + * IMPORTANT: this type *MUST* be supported by atomic instructions! + */ +typedef uint64 intctr_t; + +/* counter types */ +typedef enum statsCtrType_e { + ctrType_IntCtr, + ctrType_Int +} statsCtrType_t; + + +/* helper entity, the counter */ +typedef struct ctr_s { + uchar *name; + statsCtrType_t ctrType; + union { + intctr_t *pIntCtr; + int *pInt; + } val; + struct ctr_s *next, *prev; +} ctr_t; + +/* the statsobj object */ +struct statsobj_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *name; + pthread_mutex_t mutCtr; /* to guard counter linked-list ops */ + ctr_t *ctrRoot; /* doubly-linked list of statsobj counters */ + ctr_t *ctrLast; + /* used to link ourselves together */ + statsobj_t *prev; + statsobj_t *next; +}; + + +/* interfaces */ +BEGINinterface(statsobj) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(statsobj); + rsRetVal (*Construct)(statsobj_t **ppThis); + rsRetVal (*ConstructFinalize)(statsobj_t *pThis); + rsRetVal (*Destruct)(statsobj_t **ppThis); + rsRetVal (*SetName)(statsobj_t *pThis, uchar *name); + rsRetVal (*GetStatsLine)(statsobj_t *pThis, cstr_t **ppcstr); + rsRetVal (*GetAllStatsLines)(rsRetVal(*cb)(void*, cstr_t*), void *usrptr); + rsRetVal (*AddCounter)(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr); + rsRetVal (*EnableStats)(void); +ENDinterface(statsobj) +#define statsobjCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(statsobj); + + +/* macros to handle stats counters + * These are to be used by "counter providers". Note that we MUST + * specify the mutex name, even though at first it looks like it + * could be automatically be generated via e.g. "mut##ctr". + * Unfortunately, this does not work if counter is e.g. "pThis->ctr". + * So we decided, for clarity, to always insist on specifying the mutex + * name (after all, it's just a few more keystrokes...). + */ +#define STATSCOUNTER_DEF(ctr, mut) \ + intctr_t ctr; \ + DEF_ATOMIC_HELPER_MUT(mut) + +#define STATSCOUNTER_INIT(ctr, mut) \ + INIT_ATOMIC_HELPER_MUT(mut); \ + ctr = 0; + +#define STATSCOUNTER_INC(ctr, mut) \ + if(GatherStats) \ + ATOMIC_INC(&ctr, mut); + +#define STATSCOUNTER_DEC(ctr, mut) \ + if(GatherStats) \ + ATOMIC_DEC(&ctr, mut); + +/* the next macro works only if the variable is already guarded + * by mutex (or the users risks a wrong result). It is assumed + * that there are not concurrent operations that modify the counter. + */ +#define STATSCOUNTER_SETMAX_NOMUT(ctr, newmax) \ + if(GatherStats && ((newmax) > (ctr))) \ + ctr = newmax; + +#endif /* #ifndef INCLUDED_STATSOBJ_H */ -- cgit v1.2.3 From 6173abfbc9f59c7a5551cb6713ff182e68165f71 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Sep 2010 15:41:31 +0200 Subject: preparing for 5.7.0 --- ChangeLog | 2 +- doc/manual.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0f8b625d..9696f398 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-?? +Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters - support for systemd officially added * acquire /dev/log socket optionally from systemd diff --git a/doc/manual.html b/doc/manual.html index 882af031..08334c5d 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.5.7 (beta branch) of rsyslog. +

    This documentation is for version 5.7.0 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 2288bbb90fc6f51cebf5db81b05c6141e754ef25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Sep 2010 15:58:03 +0200 Subject: fixed compile problems on Solaris --- runtime/statsobj.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/statsobj.h b/runtime/statsobj.h index 1ab02182..7447cded 100644 --- a/runtime/statsobj.h +++ b/runtime/statsobj.h @@ -99,7 +99,7 @@ PROTOTYPEObj(statsobj); */ #define STATSCOUNTER_DEF(ctr, mut) \ intctr_t ctr; \ - DEF_ATOMIC_HELPER_MUT(mut) + DEF_ATOMIC_HELPER_MUT(mut); #define STATSCOUNTER_INIT(ctr, mut) \ INIT_ATOMIC_HELPER_MUT(mut); \ @@ -107,7 +107,7 @@ PROTOTYPEObj(statsobj); #define STATSCOUNTER_INC(ctr, mut) \ if(GatherStats) \ - ATOMIC_INC(&ctr, mut); + ATOMIC_INC(&ctr, &mut); #define STATSCOUNTER_DEC(ctr, mut) \ if(GatherStats) \ -- cgit v1.2.3 From 735be1e247563d422c18d955530eb8360e706361 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Sep 2010 10:15:20 +0200 Subject: minor: improved error message when already running --- tools/syslogd.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index 56000e08..4964f8fc 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2437,10 +2437,9 @@ doGlblProcessInit(void) for (i = 3; i < num_fds; i++) (void) close(i); untty(); - } - else - { - fputs(" Already running.\n", stderr); + } else { + fputs(" Already running. If you want to run multiple instances, you need " + "to specify different pid files (use -i option)\n", stderr); exit(1); /* "good" exit, done if syslogd is already running */ } } -- cgit v1.2.3 From 2a2cc3d26d56b834dcc143f2f4e3abbb6bd206c6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Sep 2010 12:20:59 +0200 Subject: minor: added version number to some error messages --- runtime/debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/debug.c b/runtime/debug.c index 64e251e5..5818dda9 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1310,7 +1310,7 @@ dbgGetRuntimeOptions(void) while(dbgGetRTOptNamVal(&pszOpts, &optname, &optval)) { if(!strcasecmp((char*)optname, "help")) { fprintf(stderr, - "rsyslogd runtime debug support - help requested, rsyslog terminates\n\n" + "rsyslogd " VERSION " runtime debug support - help requested, rsyslog terminates\n\n" "environment variables:\n" "addional logfile: export RSYSLOG_DEBUGFILE=\"/path/to/file\"\n" "to set: export RSYSLOG_DEBUG=\"cmd cmd cmd\"\n\n" @@ -1358,7 +1358,7 @@ dbgGetRuntimeOptions(void) bAbortTrace = 0; } else if(!strcasecmp((char*)optname, "filetrace")) { if(*optval == '\0') { - fprintf(stderr, "Error: logfile debug option requires filename, " + fprintf(stderr, "rsyslogd " VERSION " error: logfile debug option requires filename, " "e.g. \"logfile=debug.c\"\n"); exit(1); } else { @@ -1366,7 +1366,7 @@ dbgGetRuntimeOptions(void) dbgPrintNameAdd(optval, &printNameFileRoot); } } else { - fprintf(stderr, "Error: invalid debug option '%s', value '%s' - ignored\n", + fprintf(stderr, "rsyslogd " VERSION " error: invalid debug option '%s', value '%s' - ignored\n", optval, optname); } } -- cgit v1.2.3 From aa1b8f38c3611fd01710e2e3d0189c90cf9a7e9d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 18:05:12 +0200 Subject: first shot at SCM_CREDENTIALS --- plugins/imuxsock/imuxsock.c | 67 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index d976c676..9a245f88 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "dirty.h" #include "cfsysline.h" #include "unicode-helper.h" @@ -192,6 +193,7 @@ static int create_unix_socket(const char *path, int bCreatePath) { struct sockaddr_un sunx; int fd; + int one; if (path[0] == '\0') return -1; @@ -226,6 +228,7 @@ static int create_unix_socket(const char *path, int bCreatePath) return -1; } + //TOD: add logic to prepare socket (or at better place!) return fd; } } @@ -246,6 +249,24 @@ static int create_unix_socket(const char *path, int bCreatePath) close(fd); return -1; } + one = 1; +dbgprintf("imuxsock: setting socket options!\n"); + if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + close(fd); + return -1; + } + if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + close(fd); + return -1; + } + one = 1; + if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); + close(fd); + return -1; + } return fd; } @@ -264,6 +285,12 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + // TODO: here we need to mangle the raw message if we need to + // "fix up" the user pid. In the long term, it may make more sense + // to add this (somehow) to the message object (problem: at this stage, + // it is not fully available, eg no structured data). Alternatively, we + // may use a custom parser for our messages, but that doesn't play too well + // with the rest of the system. if(funixParseHost[iSock]) { pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; } else { @@ -292,7 +319,13 @@ static rsRetVal readSocket(int fd, int iSock) DEFiRet; int iRcvd; int iMaxLine; + struct msghdr msgh; + struct iovec msgiov; + static int ratelimitErrmsg = 0; // TODO: atomic OPS + struct cmsghdr *cm; + struct ucred *cred; uchar bufRcv[4096+1]; + char aux[1024]; uchar *pRcv = NULL; /* receive buffer */ assert(iSock >= 0); @@ -311,15 +344,43 @@ static rsRetVal readSocket(int fd, int iSock) CHKmalloc(pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))); } - iRcvd = recv(fd, pRcv, iMaxLine, 0); + memset(&msgh, 0, sizeof(msgh)); + memset(&msgiov, 0, sizeof(msgiov)); + memset(&aux, 0, sizeof(aux)); + msgiov.iov_base = pRcv; + msgiov.iov_len = iMaxLine; + msgh.msg_iov = &msgiov; + msgh.msg_iovlen = 1; + msgh.msg_control = aux; + msgh.msg_controllen = sizeof(aux); + iRcvd = recvmsg(fd, &msgh, MSG_DONTWAIT); +/* iRcvd = recv(fd, pRcv, iMaxLine, 0); + */ + dbgprintf("Message from UNIX socket: #%d\n", fd); if (iRcvd > 0) { + ratelimitErrmsg = 0; + + +dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); + for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { +dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { + cred = (struct ucred*) CMSG_DATA(cm); + dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); + //break; + } + } +dbgprintf("XXX: post CM loop\n"); + + + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); - } else if (iRcvd < 0 && errno != EINTR) { + } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("UNIX socket error: %d = %s.\n", errno, errStr); - errmsg.LogError(errno, NO_ERRCODE, "recvfrom UNIX"); + errmsg.LogError(errno, NO_ERRCODE, "imuxsock: recvfrom UNIX"); } finalize_it: -- cgit v1.2.3 From c2d36c6894f7c36c2aed23c1e7ee9580c58659a7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 18:57:05 +0200 Subject: patching SCM_CREDENTIALS pid into TAG of imuxsock read messages --- plugins/imuxsock/imuxsock.c | 179 ++++++++++++++++++++++++++++++++++---------- runtime/rsyslog.h | 1 + 2 files changed, 142 insertions(+), 38 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 9a245f88..fff6ac72 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -29,6 +29,7 @@ #include "rsyslog.h" #include #include +#include #include #include #include @@ -50,6 +51,7 @@ #include "unlimited_select.h" #include "sd-daemon.h" #include "statsobj.h" +#include "datetime.h" MODULE_TYPE_INPUT @@ -74,6 +76,7 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(datetime) DEFobjCurrIf(statsobj) statsobj_t *modStats; @@ -189,9 +192,38 @@ static rsRetVal discardFunixn(void) } -static int create_unix_socket(const char *path, int bCreatePath) +/* used to create a log socket if NOT passed in via systemd. + */ +static inline rsRetVal +createLogSocket(const char *path, int bCreatePath, int *fd) { struct sockaddr_un sunx; + DEFiRet; + + unlink(path); + memset(&sunx, 0, sizeof(sunx)); + sunx.sun_family = AF_UNIX; + if(bCreatePath) { + makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); + } + strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); + *fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if(*fd < 0 || bind(*fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || + chmod(path, 0666) < 0) { + errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); + dbgprintf("cannot create %s (%d).\n", path, errno); + close(*fd); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } +finalize_it: + RETiRet; +} + + +static inline rsRetVal +openLogSocket(const char *path, int bCreatePath, int *pfd) +{ + DEFiRet; int fd; int one; @@ -201,18 +233,18 @@ static int create_unix_socket(const char *path, int bCreatePath) if (strcmp(path, _PATH_LOG) == 0) { int r; - /* Check whether an FD was passed in from systemd. If + /* System log socket code. Check whether an FD was passed in from systemd. If * so, it's the /dev/log socket, so use it. */ r = sd_listen_fds(0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to acquire systemd socket"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (r > 1) { errmsg.LogError(EINVAL, NO_ERRCODE, "Wrong number of systemd sockets passed"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (r == 1) { @@ -220,46 +252,31 @@ static int create_unix_socket(const char *path, int bCreatePath) r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (!r) { errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } - - //TOD: add logic to prepare socket (or at better place!) - return fd; - } - } - - unlink(path); - - memset(&sunx, 0, sizeof(sunx)); - sunx.sun_family = AF_UNIX; - if(bCreatePath) { - makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); - } - (void) strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); - fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if (fd < 0 || bind(fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || - chmod(path, 0666) < 0) { - errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); - dbgprintf("cannot create %s (%d).\n", path, errno); - close(fd); - return -1; + } else { + CHKiRet(createLogSocket(path, bCreatePath, &fd)); + } + } else { + CHKiRet(createLogSocket(path, bCreatePath, &fd)); } + one = 1; dbgprintf("imuxsock: setting socket options!\n"); - if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } one = 1; if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { @@ -267,16 +284,53 @@ dbgprintf("imuxsock: setting socket options!\n"); close(fd); return -1; } - return fd; + *pfd = fd; +finalize_it: + RETiRet; +} + + +/* patch correct pid into tag. bufTAG MUST be CONF_TAG_MAXSIZE long! + */ +static inline void +fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) +{ + int i; + char bufPID[16]; + int lenPID; + + if(cred == NULL) + return; + + lenPID = snprintf(bufPID, sizeof(bufPID), "[%u]:", (unsigned) cred->pid); + + for(i = *lenTag ; i >= 0 && bufTAG[i] != '[' ; --i) + /*JUST SKIP*/; + + if(i < 0) + i = *lenTag - 1; /* go right at end of TAG, pid was not present (-1 for ':') */ + + if(i + lenPID > CONF_TAG_MAXSIZE) + return; /* do not touch, as things would break */ + + memcpy(bufTAG + i, bufPID, lenPID); + *lenTag = i + lenPID; } /* submit received message to the queue engine + * We now parse the message according to expected format so that we + * can also mangle it if necessary. */ static inline rsRetVal -SubmitMsg(uchar *pRcv, int lenRcv, int iSock) +SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) { msg_t *pMsg; + int lenMsg; + int i; + uchar *parse; + int pri; + uchar bufParseTAG[CONF_TAG_MAXSIZE]; DEFiRet; /* we now create our own message object and submit it to the queue */ @@ -285,6 +339,43 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); +// TODO: handle format errors + parse = pRcv; + lenMsg = lenRcv; + + parse++; lenMsg--; /* '<' */ + pri = 0; + while(lenMsg && isdigit(*parse)) { + pri = pri * 10 + *parse - '0'; + ++parse; + --lenMsg; + } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); +dbgprintf("imuxsock: submit: facil %d, sever %d\n", pMsg->iFacility, pMsg->iSeverity); + + parse++; lenMsg--; /* '>' */ + +dbgprintf("imuxsock: submit: stage 2\n"); + if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { + dbgprintf("we have a problem, invalid timestamp in msg!\n"); + } + + /* pull tag */ +dbgprintf("imuxsock: submit: stage 3\n"); + + i = 0; + while(lenMsg > 0 && *parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *parse++; + --lenMsg; + } + bufParseTAG[i] = '\0'; /* terminate string */ + fixPID(bufParseTAG, &i, cred); + MsgSetTAG(pMsg, bufParseTAG, i); + + MsgSetMSGoffs(pMsg, lenRcv - lenMsg); + // TODO: here we need to mangle the raw message if we need to // "fix up" the user pid. In the long term, it may make more sense // to add this (somehow) to the message object (problem: at this stage, @@ -292,9 +383,9 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) // may use a custom parser for our messages, but that doesn't play too well // with the rest of the system. if(funixParseHost[iSock]) { - pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->msgFlags = funixFlags[iSock] | PARSE_HOSTNAME; } else { - pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING; + pMsg->msgFlags = funixFlags[iSock]; } MsgSetRcvFrom(pMsg, funixHName[iSock]); @@ -363,6 +454,7 @@ static rsRetVal readSocket(int fd, int iSock) dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); + cred = NULL; for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { @@ -375,7 +467,7 @@ dbgprintf("XXX: post CM loop\n"); - CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock, cred)); } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -458,6 +550,7 @@ ENDrunInput BEGINwillRun CODESTARTwillRun register int i; + int actSocks; /* first apply some config settings */ # ifdef OS_SOLARIS @@ -474,9 +567,17 @@ CODESTARTwillRun funixn[0] = pLogSockName; /* initialize and return if will run or not */ + actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfunix ; i++) { - if ((funix[i] = create_unix_socket((char*) funixn[i], funixCreateSockPath[i])) != -1) + if(openLogSocket((char*) funixn[i], funixCreateSockPath[i], &(funix[i])) == RS_RET_OK) { + ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); + } + } + + if(actSocks == 0) { + errmsg.LogError(0, NO_ERRCODE, "imuxsock does not run because we could not aquire any socket\n"); + ABORT_FINALIZE(RS_RET_ERR); } /* we need to create the inputName property (only once during our lifetime) */ @@ -526,6 +627,7 @@ CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); objRelease(statsobj, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); ENDmodExit @@ -573,6 +675,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(statsobj, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 7b2655b0..c58c259b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -456,6 +456,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_EPOLL_CR_FAILED = -2173, /**< epoll_create() failed */ RS_RET_EPOLL_CTL_FAILED = -2174, /**< epoll_ctl() failed */ RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ + RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 7d680cb7590d4eefa4631a249a02248b4168c7c6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 19:05:20 +0200 Subject: fixed nit --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index fff6ac72..496e679f 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -282,7 +282,7 @@ dbgprintf("imuxsock: setting socket options!\n"); if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } *pfd = fd; finalize_it: -- cgit v1.2.3 From 43933920353ed6911738255d509cd001960ff1b7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 10:28:47 +0200 Subject: changed imuxsock to have a somewhat better representation of listeners still not great, but far better readable (and extendable) than what we had before. --- plugins/imuxsock/imuxsock.c | 156 +++++++++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 61 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 496e679f..493f11c4 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -82,20 +82,26 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +/* structure to describe a specific listener */ +struct lstn_s { + uchar *sockName; /* read-only after startup */ + prop_t *hostName; /* host-name override - if set, use this instead of actual name */ + int fd; /* read-only after startup */ + int flags; /* should parser parse host name? read-only after startup */ + int flowCtl; /* flow control settings for this socket */ + sbool bParseHost; /* should parser parse host name? read-only after startup */ + sbool bUseCreds; + sbool bCreateSockPath; /* auto-creation of socket directory? */ +}; +static struct lstn_s listeners[MAXFUNIX]; + static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -static int startIndexUxLocalSockets; /* process funix from that index on (used to +static int startIndexUxLocalSockets; /* process fd from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup */ -static int funixParseHost[MAXFUNIX] = { 0, }; /* should parser parse host name? read-only after startup */ -static int funixFlags[MAXFUNIX] = { IGNDATE, }; /* should parser parse host name? read-only after startup */ -static int funixCreateSockPath[MAXFUNIX] = { 0, }; /* auto-creation of socket directory? */ -static uchar *funixn[MAXFUNIX] = { (uchar*) _PATH_LOG }; /* read-only after startup */ -static prop_t *funixHName[MAXFUNIX] = { NULL, }; /* host-name override - if set, use this instead of actual name */ -static int funixFlowCtl[MAXFUNIX] = { eFLOWCTL_NO_DELAY, }; /* flow control settings for this socket */ -static int funix[MAXFUNIX] = { -1, }; /* read-only after startup */ -static int nfunix = 1; /* number of Unix sockets open / read-only after startup */ +static int nfd = 1; /* number of Unix sockets open / read-only after startup */ /* config settings */ static int bOmitLocalLogging = 0; @@ -103,6 +109,7 @@ static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreateSockPath 0 static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? */ @@ -114,7 +121,7 @@ static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? * static rsRetVal setSystemLogTimestampIgnore(void __attribute__((unused)) *pVal, int iNewVal) { DEFiRet; - funixFlags[0] = iNewVal ? IGNDATE : NOFLAG; + listeners[0].flags = iNewVal ? IGNDATE : NOFLAG; RETiRet; } @@ -123,7 +130,7 @@ static rsRetVal setSystemLogTimestampIgnore(void __attribute__((unused)) *pVal, static rsRetVal setSystemLogFlowControl(void __attribute__((unused)) *pVal, int iNewVal) { DEFiRet; - funixFlowCtl[0] = iNewVal ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; + listeners[0].flowCtl = iNewVal ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; RETiRet; } @@ -140,26 +147,28 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; - if(nfunix < MAXFUNIX) { + if(nfd < MAXFUNIX) { if(*pNewVal == ':') { - funixParseHost[nfunix] = 1; + listeners[nfd].bParseHost = 1; } else { - funixParseHost[nfunix] = 0; + listeners[nfd].bParseHost = 0; } - CHKiRet(prop.Construct(&(funixHName[nfunix]))); + CHKiRet(prop.Construct(&(listeners[nfd].hostName))); if(pLogHostName == NULL) { - CHKiRet(prop.SetString(funixHName[nfunix], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.SetString(listeners[nfd].hostName, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); } else { - CHKiRet(prop.SetString(funixHName[nfunix], pLogHostName, ustrlen(pLogHostName))); + CHKiRet(prop.SetString(listeners[nfd].hostName, pLogHostName, ustrlen(pLogHostName))); /* reset hostname for next socket */ free(pLogHostName); pLogHostName = NULL; } - CHKiRet(prop.ConstructFinalize(funixHName[nfunix])); - funixFlowCtl[nfunix] = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; - funixFlags[nfunix] = bIgnoreTimestamp ? IGNDATE : NOFLAG; - funixCreateSockPath[nfunix] = bCreateSockPath; - funixn[nfunix++] = pNewVal; + CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); + listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; + listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; + listeners[nfd].bCreateSockPath = bCreateSockPath; + listeners[nfd].sockName = pNewVal; + listeners[nfd].bUseCreds = bUseCreds; + nfd++; } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", pNewVal); @@ -170,21 +179,21 @@ finalize_it: } -/* free the funixn[] socket names - needed as cleanup on several places - * note that nfunix is NOT reset! funixn[0] is never freed, as it comes from +/* free the sockName[] socket names - needed as cleanup on several places + * note that nfd is NOT reset! sockName[0] is never freed, as it comes from * the constant memory pool - and if not, it is freeed via some other pointer. */ static rsRetVal discardFunixn(void) { int i; - for (i = 1; i < nfunix; i++) { - if(funixn[i] != NULL) { - free(funixn[i]); - funixn[i] = NULL; + for (i = 1; i < nfd; i++) { + if(listeners[i].sockName != NULL) { + free(listeners[i].sockName); + listeners[i].sockName = NULL; } - if(funixHName[i] != NULL) { - prop.Destruct(&(funixHName[i])); + if(listeners[i].hostName != NULL) { + prop.Destruct(&(listeners[i].hostName)); } } @@ -267,7 +276,6 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) } one = 1; -dbgprintf("imuxsock: setting socket options!\n"); if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); @@ -284,6 +292,17 @@ dbgprintf("imuxsock: setting socket options!\n"); close(fd); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } + + + +static socklen_t two =sizeof(one); + if(getsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one,&two) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); + close(fd); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } +dbgprintf("imuxsock: SO_TIMESTAMP is %d\n", one); + *pfd = fd; finalize_it: RETiRet; @@ -337,7 +356,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) CHKiRet(msgConstruct(&pMsg)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); - MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + MsgSetFlowControlType(pMsg, listeners[iSock].flowCtl); // TODO: handle format errors parse = pRcv; @@ -382,13 +401,13 @@ dbgprintf("imuxsock: submit: stage 3\n"); // it is not fully available, eg no structured data). Alternatively, we // may use a custom parser for our messages, but that doesn't play too well // with the rest of the system. - if(funixParseHost[iSock]) { - pMsg->msgFlags = funixFlags[iSock] | PARSE_HOSTNAME; + if(listeners[iSock].bParseHost) { + pMsg->msgFlags = listeners[iSock].flags | PARSE_HOSTNAME; } else { - pMsg->msgFlags = funixFlags[iSock]; + pMsg->msgFlags = listeners[iSock].flags; } - MsgSetRcvFrom(pMsg, funixHName[iSock]); + MsgSetRcvFrom(pMsg, listeners[iSock].hostName); CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); @@ -453,7 +472,7 @@ static rsRetVal readSocket(int fd, int iSock) ratelimitErrmsg = 0; -dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); +dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); cred = NULL; for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); @@ -511,10 +530,11 @@ CODESTARTrunInput maxfds = 0; FD_ZERO (pReadfds); /* Copy master connections */ - for (i = startIndexUxLocalSockets; i < nfunix; i++) { - if (funix[i] != -1) { - FD_SET(funix[i], pReadfds); - if (funix[i]>maxfds) maxfds=funix[i]; + for (i = startIndexUxLocalSockets; i < nfd; i++) { + if (listeners[i].fd!= -1) { + FD_SET(listeners[i].fd, pReadfds); + if(listeners[i].fd > maxfds) + maxfds=listeners[i].fd; } } @@ -531,10 +551,10 @@ CODESTARTrunInput if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ - for (i = 0; i < nfunix && nfds > 0; i++) { + for (i = 0; i < nfd && nfds > 0; i++) { if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ - if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) { + if ((fd = listeners[i].fd) != -1 && FD_ISSET(fd, pReadfds)) { readSocket(fd, i); --nfds; /* indicate we have processed one */ } @@ -564,14 +584,15 @@ CODESTARTwillRun startIndexUxLocalSockets = bOmitLocalLogging ? 1 : 0; # endif if(pLogSockName != NULL) - funixn[0] = pLogSockName; + listeners[0].sockName = pLogSockName; /* initialize and return if will run or not */ actSocks = 0; - for (i = startIndexUxLocalSockets ; i < nfunix ; i++) { - if(openLogSocket((char*) funixn[i], funixCreateSockPath[i], &(funix[i])) == RS_RET_OK) { + for (i = startIndexUxLocalSockets ; i < nfd ; i++) { + if(openLogSocket((char*) listeners[i].sockName, listeners[i].bCreateSockPath, + &(listeners[i].fd)) == RS_RET_OK) { ++actSocks; - dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); + dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } } @@ -594,9 +615,9 @@ CODESTARTafterRun int i; /* do cleanup here */ /* Close the UNIX sockets. */ - for (i = 0; i < nfunix; i++) - if (funix[i] != -1) - close(funix[i]); + for (i = 0; i < nfd; i++) + if (listeners[i].fd != -1) + close(listeners[i].fd); /* Clean-up files. If systemd passed us a socket it is * systemd's job to clean it up.*/ @@ -604,15 +625,15 @@ CODESTARTafterRun i = 1; else i = startIndexUxLocalSockets; - for(; i < nfunix; i++) - if (funixn[i] && funix[i] != -1) - unlink((char*) funixn[i]); + for(; i < nfd; i++) + if (listeners[i].sockName && listeners[i].fd != -1) + unlink((char*) listeners[i].sockName); /* free no longer needed string */ free(pLogSockName); free(pLogHostName); discardFunixn(); - nfunix = 1; + nfd = 1; if(pInputName != NULL) prop.Destruct(&pInputName); @@ -657,9 +678,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } discardFunixn(); - nfunix = 1; + nfd = 1; bIgnoreTimestamp = 1; bUseFlowCtl = 0; + bUseCreds = 0; bCreateSockPath = DFLT_bCreateSockPath; return RS_RET_OK; @@ -679,10 +701,20 @@ CODEmodInit_QueryRegCFSLineHdlr dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); - /* initialize funixn[] array */ + /* init system log socket settings */ + listeners[0].flags = IGNDATE; + listeners[0].sockName = UCHAR_CONSTANT(_PATH_LOG); + listeners[0].hostName = NULL; + listeners[0].flowCtl = eFLOWCTL_NO_DELAY; + listeners[0].fd = -1; + listeners[0].bParseHost = 0; + listeners[0].bUseCreds = 0; + listeners[0].bCreateSockPath = 0; + + /* initialize socket names */ for(i = 1 ; i < MAXFUNIX ; ++i) { - funixn[i] = NULL; - funix[i] = -1; + listeners[i].sockName = NULL; + listeners[i].fd = -1; } CHKiRet(prop.Construct(&pLocalHostIP)); @@ -690,9 +722,9 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(prop.ConstructFinalize(pLocalHostIP)); /* now init listen socket zero, the local log socket */ - CHKiRet(prop.Construct(&(funixHName[0]))); - CHKiRet(prop.SetString(funixHName[0], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); - CHKiRet(prop.ConstructFinalize(funixHName[0])); + CHKiRet(prop.Construct(&(listeners[0].hostName))); + CHKiRet(prop.SetString(listeners[0].hostName, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.ConstructFinalize(listeners[0].hostName)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"omitlocallogging", 0, eCmdHdlrBinary, @@ -707,6 +739,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, NULL, &bCreateSockPath, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusespidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit v1.2.3 From 3773aadcdd8008b97bb532f6d29fe29d17b06159 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 11:41:17 +0200 Subject: added support for SCM_CREDENTIALS to imuxsock (now fully working) --- ChangeLog | 7 ++ doc/imuxsock.html | 8 ++ plugins/imuxsock/imuxsock.c | 197 +++++++++++++++++++++----------------------- 3 files changed, 107 insertions(+), 105 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9696f398..6622b67c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,11 @@ --------------------------------------------------------------------------- +Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? +- imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log + socket itself (thanks to Lennart Poettering for the suggestion) +- added new config statements + * $InputUnixListenSocketUsePIDFromSystem + * $SystemLogUsePIDFromSystem +--------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters - support for systemd officially added diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 381374d2..4af2c030 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -40,12 +40,20 @@ the implications. Note that for many systems, turning on flow control does not h
    Ignore timestamps included in the message. Applies to the next socket being added.

  • $InputUnixListenSocketFlowControl [on/off] - specifies if flow control should be applied to the next socket.
    • +
  • $InputUnixListenSocketUsePIDFromSystem [on/off] - specifies if the pid being logged shall +be obtained from the log socket itself. If so, the TAG part of the message is rewritten. +It is recommended to turn this option on, but the default is "off" to keep compatible +with earlier versions of rsyslog. This option was introduced in 5.7.0.
  • $SystemLogSocketIgnoreMsgTimestamp [on/off]
    Ignore timestamps included in the messages, applies to messages received via the system log socket.
  • $OmitLocalLogging (imuxsock) [on/off] -- former -o option
  • $SystemLogSocketName <name-of-socket> -- former -p option
  • $SystemLogFlowControl [on/off] - specifies if flow control should be applied to the system log socket.
    • +
  • $SystemLogUsePIDFromSystem [on/off] - specifies if the pid being logged shall +be obtained from the log socket itself. If so, the TAG part of the message is rewritten. +It is recommended to turn this option on, but the default is "off" to keep compatible +with earlier versions of rsyslog. This option was introduced in 5.7.0.
  • $InputUnixListenSocketCreatePath [on/off] - create directories in the socket path if they do not already exist. They are created with 0755 permissions with the owner being the process under which rsyslogd runs. The default is not to create directories. Keep in mind, though, that rsyslogd always diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 493f11c4..df2b8efc 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -83,7 +83,7 @@ statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) /* structure to describe a specific listener */ -struct lstn_s { +typedef struct lstn_s { uchar *sockName; /* read-only after startup */ prop_t *hostName; /* host-name override - if set, use this instead of actual name */ int fd; /* read-only after startup */ @@ -91,9 +91,9 @@ struct lstn_s { int flowCtl; /* flow control settings for this socket */ sbool bParseHost; /* should parser parse host name? read-only after startup */ sbool bUseCreds; - sbool bCreateSockPath; /* auto-creation of socket directory? */ -}; -static struct lstn_s listeners[MAXFUNIX]; + sbool bCreatePath; /* auto-creation of socket directory? */ +} lstn_t; +static lstn_t listeners[MAXFUNIX]; static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ @@ -108,10 +108,11 @@ static int bOmitLocalLogging = 0; static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ -static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ -static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ -#define DFLT_bCreateSockPath 0 -static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? */ +static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +#define DFLT_bCreatePath 0 +static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ /* set the timestamp ignore / not ignore option for the system @@ -165,7 +166,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; - listeners[nfd].bCreateSockPath = bCreateSockPath; + listeners[nfd].bCreatePath = bCreatePath; listeners[nfd].sockName = pNewVal; listeners[nfd].bUseCreds = bUseCreds; nfd++; @@ -204,24 +205,25 @@ static rsRetVal discardFunixn(void) /* used to create a log socket if NOT passed in via systemd. */ static inline rsRetVal -createLogSocket(const char *path, int bCreatePath, int *fd) +createLogSocket(lstn_t *pLstn) { struct sockaddr_un sunx; DEFiRet; - unlink(path); + unlink((char*)pLstn->sockName); memset(&sunx, 0, sizeof(sunx)); sunx.sun_family = AF_UNIX; - if(bCreatePath) { - makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); + if(pLstn->bCreatePath) { + makeFileParentDirs((uchar*)pLstn->sockName, ustrlen(pLstn->sockName), 0755, -1, -1, 0); } - strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); - *fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if(*fd < 0 || bind(*fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || - chmod(path, 0666) < 0) { - errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); - dbgprintf("cannot create %s (%d).\n", path, errno); - close(*fd); + strncpy(sunx.sun_path, (char*)pLstn->sockName, sizeof(sunx.sun_path)); + pLstn->fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if(pLstn->fd < 0 || bind(pLstn->fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || + chmod((char*)pLstn->sockName, 0666) < 0) { + errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", pLstn->sockName); + dbgprintf("cannot create %s (%d).\n", pLstn->sockName, errno); + close(pLstn->fd); + pLstn->fd = -1; ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } finalize_it: @@ -230,16 +232,15 @@ finalize_it: static inline rsRetVal -openLogSocket(const char *path, int bCreatePath, int *pfd) +openLogSocket(lstn_t *pLstn) { DEFiRet; - int fd; int one; - if (path[0] == '\0') + if(pLstn->sockName[0] == '\0') return -1; - if (strcmp(path, _PATH_LOG) == 0) { + if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { int r; /* System log socket code. Check whether an FD was passed in from systemd. If @@ -257,8 +258,8 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) } if (r == 1) { - fd = SD_LISTEN_FDS_START; - r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); + pLstn->fd = SD_LISTEN_FDS_START; + r = sd_is_socket_unix(pLstn->fd, SOCK_DGRAM, -1, _PATH_LOG, 0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); @@ -269,42 +270,39 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } } else { - CHKiRet(createLogSocket(path, bCreatePath, &fd)); + CHKiRet(createLogSocket(pLstn)); } } else { - CHKiRet(createLogSocket(path, bCreatePath, &fd)); + CHKiRet(createLogSocket(pLstn)); } - one = 1; - if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + if(pLstn->bUseCreds) { + one = 1; + if(setsockopt(pLstn->fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_PASSCRED failed on '%s'", pLstn->sockName); + pLstn->bUseCreds = 0; + } + if(setsockopt(pLstn->fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS failed on '%s'", pLstn->sockName); + pLstn->bUseCreds = 0; + } } + +#if 0 + // SO_TIMESTAMP currently does not work for an unknown reason. Any help is appreciated! one = 1; - if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); - close(fd); + if(setsockopt(pLstn->fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", pLstn->sockName); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } +#endif - - -static socklen_t two =sizeof(one); - if(getsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one,&two) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); +finalize_it: + if(iRet != RS_RET_OK) { + close(pLstn->fd); + pLstn->fd = -1; } -dbgprintf("imuxsock: SO_TIMESTAMP is %d\n", one); - *pfd = fd; -finalize_it: RETiRet; } @@ -321,7 +319,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) if(cred == NULL) return; - lenPID = snprintf(bufPID, sizeof(bufPID), "[%u]:", (unsigned) cred->pid); + lenPID = snprintf(bufPID, sizeof(bufPID), "[%lu]:", (unsigned long) cred->pid); for(i = *lenTag ; i >= 0 && bufTAG[i] != '[' ; --i) /*JUST SKIP*/; @@ -342,7 +340,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) * can also mangle it if necessary. */ static inline rsRetVal -SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) +SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) { msg_t *pMsg; int lenMsg; @@ -356,7 +354,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) CHKiRet(msgConstruct(&pMsg)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); - MsgSetFlowControlType(pMsg, listeners[iSock].flowCtl); + MsgSetFlowControlType(pMsg, pLstn->flowCtl); // TODO: handle format errors parse = pRcv; @@ -372,17 +370,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); -dbgprintf("imuxsock: submit: facil %d, sever %d\n", pMsg->iFacility, pMsg->iSeverity); parse++; lenMsg--; /* '>' */ -dbgprintf("imuxsock: submit: stage 2\n"); if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { dbgprintf("we have a problem, invalid timestamp in msg!\n"); } /* pull tag */ -dbgprintf("imuxsock: submit: stage 3\n"); i = 0; while(lenMsg > 0 && *parse != ' ' && i < CONF_TAG_MAXSIZE) { @@ -395,19 +390,13 @@ dbgprintf("imuxsock: submit: stage 3\n"); MsgSetMSGoffs(pMsg, lenRcv - lenMsg); - // TODO: here we need to mangle the raw message if we need to - // "fix up" the user pid. In the long term, it may make more sense - // to add this (somehow) to the message object (problem: at this stage, - // it is not fully available, eg no structured data). Alternatively, we - // may use a custom parser for our messages, but that doesn't play too well - // with the rest of the system. - if(listeners[iSock].bParseHost) { - pMsg->msgFlags = listeners[iSock].flags | PARSE_HOSTNAME; + if(pLstn->bParseHost) { + pMsg->msgFlags = pLstn->flags | PARSE_HOSTNAME; } else { - pMsg->msgFlags = listeners[iSock].flags; + pMsg->msgFlags = pLstn->flags; } - MsgSetRcvFrom(pMsg, listeners[iSock].hostName); + MsgSetRcvFrom(pMsg, pLstn->hostName); CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); @@ -424,21 +413,20 @@ finalize_it: * of the socket which is to be processed. This eases access to the * growing number of properties. -- rgerhards, 2008-08-01 */ -static rsRetVal readSocket(int fd, int iSock) +static rsRetVal readSocket(lstn_t *pLstn) { DEFiRet; int iRcvd; int iMaxLine; struct msghdr msgh; struct iovec msgiov; - static int ratelimitErrmsg = 0; // TODO: atomic OPS struct cmsghdr *cm; struct ucred *cred; uchar bufRcv[4096+1]; - char aux[1024]; + char aux[128]; uchar *pRcv = NULL; /* receive buffer */ - assert(iSock >= 0); + assert(pLstn->fd >= 0); iMaxLine = glbl.GetMaxLine(); @@ -456,38 +444,34 @@ static rsRetVal readSocket(int fd, int iSock) memset(&msgh, 0, sizeof(msgh)); memset(&msgiov, 0, sizeof(msgiov)); - memset(&aux, 0, sizeof(aux)); + if(pLstn->bUseCreds) { + memset(&aux, 0, sizeof(aux)); + msgh.msg_control = aux; + msgh.msg_controllen = sizeof(aux); + } msgiov.iov_base = pRcv; msgiov.iov_len = iMaxLine; msgh.msg_iov = &msgiov; msgh.msg_iovlen = 1; - msgh.msg_control = aux; - msgh.msg_controllen = sizeof(aux); - iRcvd = recvmsg(fd, &msgh, MSG_DONTWAIT); -/* iRcvd = recv(fd, pRcv, iMaxLine, 0); - */ + iRcvd = recvmsg(pLstn->fd, &msgh, MSG_DONTWAIT); - dbgprintf("Message from UNIX socket: #%d\n", fd); - if (iRcvd > 0) { - ratelimitErrmsg = 0; - - -dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); - cred = NULL; - for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { -dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); - if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { - cred = (struct ucred*) CMSG_DATA(cm); - dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); - //break; - } - } -dbgprintf("XXX: post CM loop\n"); - - - - CHKiRet(SubmitMsg(pRcv, iRcvd, iSock, cred)); - } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { + dbgprintf("Message from UNIX socket: #%d\n", pLstn->fd); + if(iRcvd > 0) { + cred = NULL; + if(pLstn->bUseCreds) { + dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); + for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { + dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { + cred = (struct ucred*) CMSG_DATA(cm); + dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); + //break; + } + } + dbgprintf("XXX: post CM loop\n"); + } + CHKiRet(SubmitMsg(pRcv, iRcvd, pLstn, cred)); + } else if(iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("UNIX socket error: %d = %s.\n", errno, errStr); @@ -555,7 +539,7 @@ CODESTARTrunInput if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ if ((fd = listeners[i].fd) != -1 && FD_ISSET(fd, pReadfds)) { - readSocket(fd, i); + readSocket(&(listeners[i])); --nfds; /* indicate we have processed one */ } } @@ -585,12 +569,12 @@ CODESTARTwillRun # endif if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; + listeners[0].bUseCreds = bUseCredsSysSock; /* initialize and return if will run or not */ actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfd ; i++) { - if(openLogSocket((char*) listeners[i].sockName, listeners[i].bCreateSockPath, - &(listeners[i].fd)) == RS_RET_OK) { + if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } @@ -682,7 +666,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bIgnoreTimestamp = 1; bUseFlowCtl = 0; bUseCreds = 0; - bCreateSockPath = DFLT_bCreateSockPath; + bUseCredsSysSock = 0; + bCreatePath = DFLT_bCreatePath; return RS_RET_OK; } @@ -709,7 +694,7 @@ CODEmodInit_QueryRegCFSLineHdlr listeners[0].fd = -1; listeners[0].bParseHost = 0; listeners[0].bUseCreds = 0; - listeners[0].bCreateSockPath = 0; + listeners[0].bCreatePath = 0; /* initialize socket names */ for(i = 1 ; i < MAXFUNIX ; ++i) { @@ -738,9 +723,11 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketflowcontrol", 0, eCmdHdlrBinary, NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, - NULL, &bCreateSockPath, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusespidfromsystem", 0, eCmdHdlrBinary, + NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit v1.2.3 From b5da6352830e9841dd367b8490d79461adb5cb22 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 15:06:30 +0200 Subject: first shot at imuxsock ratelimiting works, but at a global level, need to go down to pid or cgroup --- plugins/imuxsock/imuxsock.c | 98 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index df2b8efc..ccd31b44 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -81,6 +81,15 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +STATSCOUNTER_DEF(ctrLostRatelimit, mutCtrLostRatelimit) + +struct rs_ratelimit_state { + unsigned short interval; + unsigned short burst; + unsigned done; + unsigned missed; + time_t begin; +} ratelimiter; /* structure to describe a specific listener */ typedef struct lstn_s { @@ -113,6 +122,70 @@ static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TA static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ +#define DFLT_ratelimitInterval 2 +static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ +#define DFLT_ratelimitBurst 200 +static int ratelimitBurst = DFLT_ratelimitBurst; /* max nbr of messages in interval */ + + + +static void +initRatelimitState(struct rs_ratelimit_state *rs, unsigned short interval, unsigned short burst) +{ + rs->interval = interval; + rs->burst = burst; + rs->done = 0; + rs->missed = 0; + rs->begin = 0; +} + + +/* ratelimiting support, modelled after the linux kernel + * returns 1 if message is within rate limit and shall be + * processed, 0 otherwise. + * This implementation is NOT THREAD-SAFE and must not + * be called concurrently. + */ +static inline int +withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) +{ + int ret; + uchar msgbuf[1024]; + + if(rs->interval == 0) { + ret = 1; + goto finalize_it; + } + + assert(rs->burst != 0); + + if(rs->begin == 0) + rs->begin = tt; + + /* resume if we go out of out time window */ + if(tt > rs->begin + rs->interval) { + if(rs->missed) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "imuxsock lost %u messages due to rate-limiting", rs->missed); + logmsgInternal(0, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + rs->missed = 0; + } + rs->begin = 0; + rs->done = 0; + } + + /* do actual limit check */ + if(rs->burst > rs->done) { + rs->done++; + ret = 1; + } else { + rs->missed++; + ret = 0; + } + +finalize_it: + return ret; +} /* set the timestamp ignore / not ignore option for the system @@ -348,14 +421,23 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) uchar *parse; int pri; uchar bufParseTAG[CONF_TAG_MAXSIZE]; + struct syslogTime st; + time_t tt; DEFiRet; + datetime.getCurrTime(&st, &tt); + if(!withinRatelimit(&ratelimiter, tt)) { + STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); + FINALIZE; + } + /* we now create our own message object and submit it to the queue */ - CHKiRet(msgConstruct(&pMsg)); + CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); + // TODO: handle format errors parse = pRcv; lenMsg = lenRcv; @@ -504,6 +586,8 @@ CODESTARTrunInput * signalled to do so. This, however, is handled by the framework, * right into the sleep below. */ + initRatelimitState(&ratelimiter, ratelimitInterval, ratelimitBurst); + while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -668,6 +752,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bUseCreds = 0; bUseCredsSysSock = 0; bCreatePath = DFLT_bCreatePath; + ratelimitInterval = DFLT_ratelimitInterval; + ratelimitBurst = DFLT_ratelimitBurst; return RS_RET_OK; } @@ -726,10 +812,12 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitinterval", 0, eCmdHdlrInt, + NULL, &ratelimitInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitburst", 0, eCmdHdlrInt, + NULL, &ratelimitBurst, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); /* the following one is a (dirty) trick: the system log socket is not added via @@ -742,12 +830,16 @@ CODEmodInit_QueryRegCFSLineHdlr setSystemLogTimestampIgnore, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); /* support statistics gathering */ CHKiRet(statsobj.Construct(&modStats)); CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), ctrType_IntCtr, &ctrSubmit)); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("lost.ratelimit"), + ctrType_IntCtr, &ctrLostRatelimit)); CHKiRet(statsobj.ConstructFinalize(modStats)); ENDmodInit -- cgit v1.2.3 From d748c68c09ae7d4f8c22c4245e5509019c59511c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 11:32:48 +0200 Subject: added some generic hashtable code by Christopher Clark found at http://www.cl.cam.ac.uk/~cwc22/hashtable/ --- plugins/imuxsock/Makefile.am | 2 +- plugins/imuxsock/imuxsock.c | 1 + runtime/Makefile.am | 2 + runtime/hashtable/Makefile | 26 ++++ runtime/hashtable/hashtable.c | 275 ++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.h | 199 ++++++++++++++++++++++++ runtime/hashtable/hashtable_itr.c | 188 +++++++++++++++++++++++ runtime/hashtable/hashtable_itr.h | 112 ++++++++++++++ runtime/hashtable/hashtable_private.h | 85 +++++++++++ runtime/hashtable/hashtable_utility.c | 71 +++++++++ runtime/hashtable/hashtable_utility.h | 55 +++++++ runtime/hashtable/tester.c | 270 +++++++++++++++++++++++++++++++++ 12 files changed, 1285 insertions(+), 1 deletion(-) create mode 100644 runtime/hashtable/Makefile create mode 100644 runtime/hashtable/hashtable.c create mode 100644 runtime/hashtable/hashtable.h create mode 100644 runtime/hashtable/hashtable_itr.c create mode 100644 runtime/hashtable/hashtable_itr.h create mode 100644 runtime/hashtable/hashtable_private.h create mode 100644 runtime/hashtable/hashtable_utility.c create mode 100644 runtime/hashtable/hashtable_utility.h create mode 100644 runtime/hashtable/tester.c diff --git a/plugins/imuxsock/Makefile.am b/plugins/imuxsock/Makefile.am index 28f9f9e3..34a0ad9a 100644 --- a/plugins/imuxsock/Makefile.am +++ b/plugins/imuxsock/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imuxsock.la imuxsock_la_SOURCES = imuxsock.c -imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I../../runtime/hashtable -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imuxsock_la_LDFLAGS = -module -avoid-version imuxsock_la_LIBADD = $(RSRT_LIBS) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index ccd31b44..b53bb379 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -52,6 +52,7 @@ #include "sd-daemon.h" #include "statsobj.h" #include "datetime.h" +#include "hashtable.h" MODULE_TYPE_INPUT diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 01b224e7..d657b6ca 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -95,6 +95,8 @@ librsyslog_la_SOURCES = \ ../parse.c \ ../parse.h \ \ + hashtable/hashtable.c \ + \ ../outchannel.c \ ../outchannel.h \ ../template.c \ diff --git a/runtime/hashtable/Makefile b/runtime/hashtable/Makefile new file mode 100644 index 00000000..3b7b5e9f --- /dev/null +++ b/runtime/hashtable/Makefile @@ -0,0 +1,26 @@ + +tester: hashtable.o tester.o hashtable_itr.o + gcc -g -Wall -O -lm -o tester hashtable.o hashtable_itr.o tester.o + +all: tester old_tester + +tester.o: tester.c + gcc -g -Wall -O -c tester.c -o tester.o + +old_tester: hashtable_powers.o tester.o hashtable_itr.o + gcc -g -Wall -O -o old_tester hashtable_powers.o hashtable_itr.o tester.o + +hashtable_powers.o: hashtable_powers.c + gcc -g -Wall -O -c hashtable_powers.c -o hashtable_powers.o + +hashtable.o: hashtable.c + gcc -g -Wall -O -c hashtable.c -o hashtable.o + +hashtable_itr.o: hashtable_itr.c + gcc -g -Wall -O -c hashtable_itr.c -o hashtable_itr.o + +tidy: + rm *.o + +clean: tidy + rm -f tester old_tester diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c new file mode 100644 index 00000000..a10e3bc6 --- /dev/null +++ b/runtime/hashtable/hashtable.c @@ -0,0 +1,275 @@ +/* Copyright (C) 2004 Christopher Clark */ +/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include +#include +#include +#include + +/* +Credit for primes table: Aaron Krowne + http://br.endernet.org/~akrowne/ + http://planetmath.org/encyclopedia/GoodHashTablePrimes.html +*/ +static const unsigned int primes[] = { +53, 97, 193, 389, +769, 1543, 3079, 6151, +12289, 24593, 49157, 98317, +196613, 393241, 786433, 1572869, +3145739, 6291469, 12582917, 25165843, +50331653, 100663319, 201326611, 402653189, +805306457, 1610612741 +}; +const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); +const float max_load_factor = 0.65; + +/*****************************************************************************/ +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashf) (void*), + int (*eqf) (void*,void*)) +{ + struct hashtable *h; + unsigned int pindex, size = primes[0]; + /* Check requested hashtable isn't too large */ + if (minsize > (1u << 30)) return NULL; + /* Enforce size as prime */ + for (pindex=0; pindex < prime_table_length; pindex++) { + if (primes[pindex] > minsize) { size = primes[pindex]; break; } + } + h = (struct hashtable *)malloc(sizeof(struct hashtable)); + if (NULL == h) return NULL; /*oom*/ + h->table = (struct entry **)malloc(sizeof(struct entry*) * size); + if (NULL == h->table) { free(h); return NULL; } /*oom*/ + memset(h->table, 0, size * sizeof(struct entry *)); + h->tablelength = size; + h->primeindex = pindex; + h->entrycount = 0; + h->hashfn = hashf; + h->eqfn = eqf; + h->loadlimit = (unsigned int) ceil(size * max_load_factor); + return h; +} + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k) +{ + /* Aim to protect against poor hash functions by adding logic here + * - logic taken from java 1.4 hashtable source */ + unsigned int i = h->hashfn(k); + i += ~(i << 9); + i ^= ((i >> 14) | (i << 18)); /* >>> */ + i += (i << 4); + i ^= ((i >> 10) | (i << 22)); /* >>> */ + return i; +} + +/*****************************************************************************/ +static int +hashtable_expand(struct hashtable *h) +{ + /* Double the size of the table to accomodate more entries */ + struct entry **newtable; + struct entry *e; + struct entry **pE; + unsigned int newsize, i, idx; + /* Check we're not hitting max capacity */ + if (h->primeindex == (prime_table_length - 1)) return 0; + newsize = primes[++(h->primeindex)]; + + newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); + if (NULL != newtable) + { + memset(newtable, 0, newsize * sizeof(struct entry *)); + /* This algorithm is not 'stable'. ie. it reverses the list + * when it transfers entries between the tables */ + for (i = 0; i < h->tablelength; i++) { + while (NULL != (e = h->table[i])) { + h->table[i] = e->next; + idx = indexFor(newsize,e->h); + e->next = newtable[idx]; + newtable[idx] = e; + } + } + free(h->table); + h->table = newtable; + } + /* Plan B: realloc instead */ + else + { + newtable = (struct entry **) + realloc(h->table, newsize * sizeof(struct entry *)); + if (NULL == newtable) { (h->primeindex)--; return 0; } + h->table = newtable; + memset(newtable[h->tablelength], 0, newsize - h->tablelength); + for (i = 0; i < h->tablelength; i++) { + for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { + idx = indexFor(newsize,e->h); + if (idx == i) + { + pE = &(e->next); + } + else + { + *pE = e->next; + e->next = newtable[idx]; + newtable[idx] = e; + } + } + } + } + h->tablelength = newsize; + h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + return -1; +} + +/*****************************************************************************/ +unsigned int +hashtable_count(struct hashtable *h) +{ + return h->entrycount; +} + +/*****************************************************************************/ +int +hashtable_insert(struct hashtable *h, void *k, void *v) +{ + /* This method allows duplicate keys - but they shouldn't be used */ + unsigned int idx; + struct entry *e; + if (++(h->entrycount) > h->loadlimit) + { + /* Ignore the return value. If expand fails, we should + * still try cramming just this value into the existing table + * -- we may not have memory for a larger table, but one more + * element may be ok. Next time we insert, we'll try expanding again.*/ + hashtable_expand(h); + } + e = (struct entry *)malloc(sizeof(struct entry)); + if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ + e->h = hash(h,k); + idx = indexFor(h->tablelength,e->h); + e->k = k; + e->v = v; + e->next = h->table[idx]; + h->table[idx] = e; + return -1; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_search(struct hashtable *h, void *k) +{ + struct entry *e; + unsigned int hashvalue, idx; + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hashvalue); + e = h->table[idx]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_remove(struct hashtable *h, void *k) +{ + /* TODO: consider compacting the table when the load factor drops enough, + * or provide a 'compact' method. */ + + struct entry *e; + struct entry **pE; + void *v; + unsigned int hashvalue, idx; + + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hash(h,k)); + pE = &(h->table[idx]); + e = *pE; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + *pE = e->next; + h->entrycount--; + v = e->v; + freekey(e->k); + free(e); + return v; + } + pE = &(e->next); + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +/* destroy */ +void +hashtable_destroy(struct hashtable *h, int free_values) +{ + unsigned int i; + struct entry *e, *f; + struct entry **table = h->table; + if (free_values) + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } + } + } + else + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f); } + } + } + free(h->table); + free(h); +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.h b/runtime/hashtable/hashtable.h new file mode 100644 index 00000000..b90781ab --- /dev/null +++ b/runtime/hashtable/hashtable.h @@ -0,0 +1,199 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_H__ +#define __HASHTABLE_CWC22_H__ + +struct hashtable; + +/* Example of use: + * + * struct hashtable *h; + * struct some_key *k; + * struct some_value *v; + * + * static unsigned int hash_from_key_fn( void *k ); + * static int keys_equal_fn ( void *key1, void *key2 ); + * + * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); + * k = (struct some_key *) malloc(sizeof(struct some_key)); + * v = (struct some_value *) malloc(sizeof(struct some_value)); + * + * (initialise k and v to suitable values) + * + * if (! hashtable_insert(h,k,v) ) + * { exit(-1); } + * + * if (NULL == (found = hashtable_search(h,k) )) + * { printf("not found!"); } + * + * if (NULL == (found = hashtable_remove(h,k) )) + * { printf("Not found\n"); } + * + */ + +/* Macros may be used to define type-safe(r) hashtable access functions, with + * methods specialized to take known key and value types as parameters. + * + * Example: + * + * Insert this at the start of your file: + * + * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); + * + * This defines the functions 'insert_some', 'search_some' and 'remove_some'. + * These operate just like hashtable_insert etc., with the same parameters, + * but their function signatures have 'struct some_key *' rather than + * 'void *', and hence can generate compile time errors if your program is + * supplying incorrect data as a key (and similarly for value). + * + * Note that the hash and key equality functions passed to create_hashtable + * still take 'void *' parameters instead of 'some key *'. This shouldn't be + * a difficult issue as they're only defined and passed once, and the other + * functions will ensure that only valid keys are supplied to them. + * + * The cost for this checking is increased code size and runtime overhead + * - if performance is important, it may be worth switching back to the + * unsafe methods once your program has been debugged with the safe methods. + * This just requires switching to some simple alternative defines - eg: + * #define insert_some hashtable_insert + * + */ + +/***************************************************************************** + * create_hashtable + + * @name create_hashtable + * @param minsize minimum initial size of hashtable + * @param hashfunction function for hashing keys + * @param key_eq_fn function for determining key equality + * @return newly created hashtable or NULL on failure + */ + +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashfunction) (void*), + int (*key_eq_fn) (void*,void*)); + +/***************************************************************************** + * hashtable_insert + + * @name hashtable_insert + * @param h the hashtable to insert into + * @param k the key - hashtable claims ownership and will free on removal + * @param v the value - does not claim ownership + * @return non-zero for successful insertion + * + * This function will cause the table to expand if the insertion would take + * the ratio of entries to table size over the maximum load factor. + * + * This function does not check for repeated insertions with a duplicate key. + * The value returned when using a duplicate key is undefined -- when + * the hashtable changes size, the order of retrieval of duplicate key + * entries is reversed. + * If in doubt, remove before insert. + */ + +int +hashtable_insert(struct hashtable *h, void *k, void *v); + +#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ +int fnname (struct hashtable *h, keytype *k, valuetype *v) \ +{ \ + return hashtable_insert(h,k,v); \ +} + +/***************************************************************************** + * hashtable_search + + * @name hashtable_search + * @param h the hashtable to search + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * +hashtable_search(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_search(h,k)); \ +} + +/***************************************************************************** + * hashtable_remove + + * @name hashtable_remove + * @param h the hashtable to remove the item from + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * /* returns value */ +hashtable_remove(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_remove(h,k)); \ +} + + +/***************************************************************************** + * hashtable_count + + * @name hashtable_count + * @param h the hashtable + * @return the number of items stored in the hashtable + */ +unsigned int +hashtable_count(struct hashtable *h); + + +/***************************************************************************** + * hashtable_destroy + + * @name hashtable_destroy + * @param h the hashtable + * @param free_values whether to call 'free' on the remaining values + */ + +void +hashtable_destroy(struct hashtable *h, int free_values); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_itr.c b/runtime/hashtable/hashtable_itr.c new file mode 100644 index 00000000..5dced841 --- /dev/null +++ b/runtime/hashtable/hashtable_itr.c @@ -0,0 +1,188 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_itr.h" +#include /* defines NULL */ + +/*****************************************************************************/ +/* hashtable_iterator - iterator constructor */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h) +{ + unsigned int i, tablelength; + struct hashtable_itr *itr = (struct hashtable_itr *) + malloc(sizeof(struct hashtable_itr)); + if (NULL == itr) return NULL; + itr->h = h; + itr->e = NULL; + itr->parent = NULL; + tablelength = h->tablelength; + itr->index = tablelength; + if (0 == h->entrycount) return itr; + + for (i = 0; i < tablelength; i++) + { + if (NULL != h->table[i]) + { + itr->e = h->table[i]; + itr->index = i; + break; + } + } + return itr; +} + +/*****************************************************************************/ +/* key - return the key of the (key,value) pair at the current position */ +/* value - return the value of the (key,value) pair at the current position */ + +void * +hashtable_iterator_key(struct hashtable_itr *i) +{ return i->e->k; } + +void * +hashtable_iterator_value(struct hashtable_itr *i) +{ return i->e->v; } + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr) +{ + unsigned int j,tablelength; + struct entry **table; + struct entry *next; + if (NULL == itr->e) return 0; /* stupidity check */ + + next = itr->e->next; + if (NULL != next) + { + itr->parent = itr->e; + itr->e = next; + return -1; + } + tablelength = itr->h->tablelength; + itr->parent = NULL; + if (tablelength <= (j = ++(itr->index))) + { + itr->e = NULL; + return 0; + } + table = itr->h->table; + while (NULL == (next = table[j])) + { + if (++j >= tablelength) + { + itr->index = tablelength; + itr->e = NULL; + return 0; + } + } + itr->index = j; + itr->e = next; + return -1; +} + +/*****************************************************************************/ +/* remove - remove the entry at the current iterator position + * and advance the iterator, if there is a successive + * element. + * If you want the value, read it before you remove: + * beware memory leaks if you don't. + * Returns zero if end of iteration. */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr) +{ + struct entry *remember_e, *remember_parent; + int ret; + + /* Do the removal */ + if (NULL == (itr->parent)) + { + /* element is head of a chain */ + itr->h->table[itr->index] = itr->e->next; + } else { + /* element is mid-chain */ + itr->parent->next = itr->e->next; + } + /* itr->e is now outside the hashtable */ + remember_e = itr->e; + itr->h->entrycount--; + freekey(remember_e->k); + + /* Advance the iterator, correcting the parent */ + remember_parent = itr->parent; + ret = hashtable_iterator_advance(itr); + if (itr->parent == remember_e) { itr->parent = remember_parent; } + free(remember_e); + return ret; +} + +/*****************************************************************************/ +int /* returns zero if not found */ +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k) +{ + struct entry *e, *parent; + unsigned int hashvalue, index; + + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + + e = h->table[index]; + parent = NULL; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + itr->index = index; + itr->e = e; + itr->parent = parent; + itr->h = h; + return -1; + } + parent = e; + e = e->next; + } + return 0; +} + + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_itr.h b/runtime/hashtable/hashtable_itr.h new file mode 100644 index 00000000..eea699a7 --- /dev/null +++ b/runtime/hashtable/hashtable_itr.h @@ -0,0 +1,112 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_ITR_CWC22__ +#define __HASHTABLE_ITR_CWC22__ +#include "hashtable.h" +#include "hashtable_private.h" /* needed to enable inlining */ + +/*****************************************************************************/ +/* This struct is only concrete here to allow the inlining of two of the + * accessor functions. */ +struct hashtable_itr +{ + struct hashtable *h; + struct entry *e; + struct entry *parent; + unsigned int index; +}; + + +/*****************************************************************************/ +/* hashtable_iterator + */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h); + +/*****************************************************************************/ +/* hashtable_iterator_key + * - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_key(struct hashtable_itr *i) +{ + return i->e->k; +} + +/*****************************************************************************/ +/* value - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_value(struct hashtable_itr *i) +{ + return i->e->v; +} + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* remove - remove current element and advance the iterator to the next element + * NB: if you need the value to free it, read it before + * removing. ie: beware memory leaks! + * returns zero if advanced to end of table */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* search - overwrite the supplied iterator, to point to the entry + * matching the supplied key. + h points to the hashtable to be searched. + * returns zero if not found. */ +int +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ +int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ +{ \ + return (hashtable_iterator_search(i,h,k)); \ +} + + + +#endif /* __HASHTABLE_ITR_CWC22__*/ + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h new file mode 100644 index 00000000..3e95f600 --- /dev/null +++ b/runtime/hashtable/hashtable_private.h @@ -0,0 +1,85 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_PRIVATE_CWC22_H__ +#define __HASHTABLE_PRIVATE_CWC22_H__ + +#include "hashtable.h" + +/*****************************************************************************/ +struct entry +{ + void *k, *v; + unsigned int h; + struct entry *next; +}; + +struct hashtable { + unsigned int tablelength; + struct entry **table; + unsigned int entrycount; + unsigned int loadlimit; + unsigned int primeindex; + unsigned int (*hashfn) (void *k); + int (*eqfn) (void *k1, void *k2); +}; + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k); + +/*****************************************************************************/ +/* indexFor */ +static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) { + return (hashvalue % tablelength); +}; + +/* Only works if tablelength == 2^N */ +/*static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) +{ + return (hashvalue & (tablelength - 1u)); +} +*/ + +/*****************************************************************************/ +#define freekey(X) free(X) +/*define freekey(X) ; */ + + +/*****************************************************************************/ + +#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_utility.c b/runtime/hashtable/hashtable_utility.c new file mode 100644 index 00000000..c3176709 --- /dev/null +++ b/runtime/hashtable/hashtable_utility.c @@ -0,0 +1,71 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_utility.h" +#include +#include +#include + +/*****************************************************************************/ +/* hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * */ +int +hashtable_change(struct hashtable *h, void *k, void *v) +{ + struct entry *e; + unsigned int hashvalue, index; + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + e = h->table[index]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + free(e->v); + e->v = v; + return -1; + } + e = e->next; + } + return 0; +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_utility.h b/runtime/hashtable/hashtable_utility.h new file mode 100644 index 00000000..56a0ffd1 --- /dev/null +++ b/runtime/hashtable/hashtable_utility.h @@ -0,0 +1,55 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_UTILITY_H__ +#define __HASHTABLE_CWC22_UTILITY_H__ + +/***************************************************************************** + * hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * @name hashtable_change + * @param h the hashtable + * @param key + * @param value + * + */ +int +hashtable_change(struct hashtable *h, void *k, void *v); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/tester.c b/runtime/hashtable/tester.c new file mode 100644 index 00000000..4678ffa8 --- /dev/null +++ b/runtime/hashtable/tester.c @@ -0,0 +1,270 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_itr.h" +#include +#include +#include /* for memcmp */ + +static const int ITEM_COUNT = 4000; + +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; + +/*****************************************************************************/ +struct key +{ + uint32_t one_ip; uint32_t two_ip; uint16_t one_port; uint16_t two_port; +}; + +struct value +{ + char *id; +}; + +DEFINE_HASHTABLE_INSERT(insert_some, struct key, struct value); +DEFINE_HASHTABLE_SEARCH(search_some, struct key, struct value); +DEFINE_HASHTABLE_REMOVE(remove_some, struct key, struct value); +DEFINE_HASHTABLE_ITERATOR_SEARCH(search_itr_some, struct key); + + +/*****************************************************************************/ +static unsigned int +hashfromkey(void *ky) +{ + struct key *k = (struct key *)ky; + return (((k->one_ip << 17) | (k->one_ip >> 15)) ^ k->two_ip) + + (k->one_port * 17) + (k->two_port * 13 * 29); +} + +static int +equalkeys(void *k1, void *k2) +{ + return (0 == memcmp(k1,k2,sizeof(struct key))); +} + +/*****************************************************************************/ +int +main(int argc, char **argv) +{ + struct key *k, *kk; + struct value *v, *found; + struct hashtable *h; + struct hashtable_itr *itr; + int i; + + h = create_hashtable(16, hashfromkey, equalkeys); + if (NULL == h) exit(-1); /*oom*/ + + +/*****************************************************************************/ +/* Insertion */ + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) exit(-1); /*oom*/ + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable search */ + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = search_some(h,k))) { + printf("BUG: key not found\n"); + } + } + +/*****************************************************************************/ +/* Hashtable iteration */ + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(h); + i = 0; + if (hashtable_count(h) > 0) + { + do { + kk = hashtable_iterator_key(itr); + v = hashtable_iterator_value(itr); + /* here (kk,v) are a valid (key, value) pair */ + /* We could call 'hashtable_remove(h,kk)' - and this operation + * 'free's kk. However, the iterator is then broken. + * This is why hashtable_iterator_remove exists - see below. + */ + i++; + + } while (hashtable_iterator_advance(itr)); + } + printf("Iterated through %u entries.\n", i); + +/*****************************************************************************/ +/* Hashtable iterator search */ + + /* Try the search some method */ + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr,h,k)) { + printf("BUG: key not found searching with iterator"); + } + } + +/*****************************************************************************/ +/* Hashtable removal */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = remove_some(h,k))) { + printf("BUG: key not found for removal\n"); + } + } + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy and create */ + + hashtable_destroy(h, 1); + h = NULL; + free(k); + + h = create_hashtable(160, hashfromkey, equalkeys); + if (NULL == h) { + printf("out of memory allocating second hashtable\n"); + return 1; + } + +/*****************************************************************************/ +/* Hashtable insertion */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) + { + printf("out of memory inserting into second hashtable\n"); + return 1; + } + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable iterator search and iterator remove */ + + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = ITEM_COUNT - 1; i >= 0; i = i - 7) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr, h, k)) { + printf("BUG: key %u not found for search preremoval using iterator\n", i); + return 1; + } + if (0 == hashtable_iterator_remove(itr)) { + printf("BUG: key not found for removal using iterator\n"); + return 1; + } + } + free(itr); + +/*****************************************************************************/ +/* Hashtable iterator remove and advance */ + + for (itr = hashtable_iterator(h); + hashtable_iterator_remove(itr) != 0; ) { + ; + } + free(itr); + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy */ + + hashtable_destroy(h, 1); + free(k); + return 0; +} + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ -- cgit v1.2.3 From 01a8807174d91a7936345d1172a87f98bbba61c4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 12:13:19 +0200 Subject: imuxsock: changed to per-pid ratelimiting --- plugins/imuxsock/imuxsock.c | 83 +++++++++++++++++++++++++++++++++++++++++---- runtime/hashtable/README | 11 ++++++ tools/Makefile.am | 2 +- 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 runtime/hashtable/README diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index b53bb379..728456b6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-20 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -83,6 +83,7 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) STATSCOUNTER_DEF(ctrLostRatelimit, mutCtrLostRatelimit) +STATSCOUNTER_DEF(ctrNumRatelimiters, mutCtrNumRatelimiters) struct rs_ratelimit_state { unsigned short interval; @@ -90,7 +91,30 @@ struct rs_ratelimit_state { unsigned done; unsigned missed; time_t begin; -} ratelimiter; +}; +typedef struct rs_ratelimit_state rs_ratelimit_state_t; + +/* support for per-process ratelimiting */ +static struct hashtable *hashtab; + +/* a very simple "hash function" for process IDs - we simply use the + * pid itself: it is quite expected that all pids may log some time, but + * from a collision point of view it is likely that long-running daemons + * start early and so will stay right in the top spots of the + * collision list. + */ +static unsigned int +hash_from_key_fn(void *k) +{ + return((unsigned) *((pid_t*) k)); +} + +static int +key_equals_fn(void *key1, void *key2) +{ + return *((pid_t*) key1) == *((pid_t*) key2); +} + /* structure to describe a specific listener */ typedef struct lstn_s { @@ -381,6 +405,43 @@ finalize_it: } +/* find ratelimiter to use for this message. Currently, we use the + * pid, but may change to cgroup later (probably via a config switch). + * Returns NULL if not found. + */ +static inline rsRetVal +findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) +{ + rs_ratelimit_state_t *rl; + int r; + pid_t *keybuf; + DEFiRet; + + if(cred == NULL) + FINALIZE; + + rl = hashtable_search(hashtab, &cred->pid); + if(rl == NULL) { + /* we need to add a new ratelimiter, process not seen before! */ + dbgprintf("imuxsock: no ratelimiter for pid %lu, creating one\n", + (unsigned long) cred->pid); + STATSCOUNTER_INC(ctrNumRatelimiters, mutCtrNumRatelimiters); + CHKmalloc(rl = malloc(sizeof(rs_ratelimit_state_t))); + CHKmalloc(keybuf = malloc(sizeof(pid_t))); + *keybuf = cred->pid; + initRatelimitState(rl, ratelimitInterval, ratelimitBurst); + r = hashtable_insert(hashtab, keybuf, rl); + if(r == 0) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + + *prl = rl; + +finalize_it: + RETiRet; +} + + /* patch correct pid into tag. bufTAG MUST be CONF_TAG_MAXSIZE long! */ static inline void @@ -424,10 +485,13 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) uchar bufParseTAG[CONF_TAG_MAXSIZE]; struct syslogTime st; time_t tt; + rs_ratelimit_state_t *ratelimiter; DEFiRet; + findRatelimiter(cred, &ratelimiter); /* ignore error, better so than others... */ + datetime.getCurrTime(&st, &tt); - if(!withinRatelimit(&ratelimiter, tt)) { + if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); FINALIZE; } @@ -587,8 +651,6 @@ CODESTARTrunInput * signalled to do so. This, however, is handled by the framework, * right into the sleep below. */ - initRatelimitState(&ratelimiter, ratelimitInterval, ratelimitBurst); - while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -675,6 +737,8 @@ CODESTARTwillRun CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); + CHKmalloc(hashtab = create_hashtable(1000, hash_from_key_fn, key_equals_fn)); + finalize_it: ENDwillRun @@ -706,6 +770,11 @@ CODESTARTafterRun if(pInputName != NULL) prop.Destruct(&pInputName); + + if(hashtab != NULL) { + hashtable_destroy(hashtab, 1); /* 1 => free all values automatically */ + hashtab = NULL; + } ENDafterRun @@ -839,8 +908,10 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), ctrType_IntCtr, &ctrSubmit)); - CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("lost.ratelimit"), + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("ratelimit.discarded"), ctrType_IntCtr, &ctrLostRatelimit)); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("ratelimit.numratelimiters"), + ctrType_IntCtr, &ctrNumRatelimiters)); CHKiRet(statsobj.ConstructFinalize(modStats)); ENDmodInit diff --git a/runtime/hashtable/README b/runtime/hashtable/README new file mode 100644 index 00000000..5cadde0c --- /dev/null +++ b/runtime/hashtable/README @@ -0,0 +1,11 @@ +This is the hashtable code provided by +Christopher Clark +available at http://www.cl.cam.ac.uk/~cwc22/hashtable/ + +It may be slightly modified. The plan is to streamline +the code based on our needs and "really" integrate it into +the rsyslog runtime library. For the time being, we use it from +inside this subdirectory. We do not need all files, but I thought +I keep them together in case we later need something else. + +rgerhards, 2010-09-28 diff --git a/tools/Makefile.am b/tools/Makefile.am index 96657ad4..6541194a 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -36,7 +36,7 @@ rsyslogd_SOURCES = \ \ ../dirty.h rsyslogd_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) +rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) -lm rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS -- cgit v1.2.3 From 054d2ccdd6044f94823f8facbda935cb70646333 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 17:26:28 +0200 Subject: imuxsock: added per-socket hash tables/rate limiters & severity filter rate limiting now applies only to messages with a given severity or above. By default, emergency messages are NOT rate-limited. --- ChangeLog | 1 + plugins/imuxsock/imuxsock.c | 159 +++++++++++++++++++++++++++++--------------- runtime/rsyslog.h | 1 + tests/syslog_caller.c | 20 ++++++ 4 files changed, 126 insertions(+), 55 deletions(-) create mode 100644 tests/syslog_caller.c diff --git a/ChangeLog b/ChangeLog index 6622b67c..ca1428b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - added new config statements * $InputUnixListenSocketUsePIDFromSystem * $SystemLogUsePIDFromSystem +- imuxsock now supports up to 50 different sockets for input --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 728456b6..d500fc54 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -57,7 +57,7 @@ MODULE_TYPE_INPUT /* defines */ -#define MAXFUNIX 20 +#define MAXFUNIX 50 #ifndef _PATH_LOG #ifdef BSD #define _PATH_LOG "/var/run/log" @@ -94,8 +94,6 @@ struct rs_ratelimit_state { }; typedef struct rs_ratelimit_state rs_ratelimit_state_t; -/* support for per-process ratelimiting */ -static struct hashtable *hashtab; /* a very simple "hash function" for process IDs - we simply use the * pid itself: it is quite expected that all pids may log some time, but @@ -118,14 +116,19 @@ key_equals_fn(void *key1, void *key2) /* structure to describe a specific listener */ typedef struct lstn_s { - uchar *sockName; /* read-only after startup */ - prop_t *hostName; /* host-name override - if set, use this instead of actual name */ - int fd; /* read-only after startup */ - int flags; /* should parser parse host name? read-only after startup */ - int flowCtl; /* flow control settings for this socket */ - sbool bParseHost; /* should parser parse host name? read-only after startup */ - sbool bUseCreds; - sbool bCreatePath; /* auto-creation of socket directory? */ + uchar *sockName; /* read-only after startup */ + prop_t *hostName; /* host-name override - if set, use this instead of actual name */ + int fd; /* read-only after startup */ + int flags; /* should parser parse host name? read-only after startup */ + int flowCtl; /* flow control settings for this socket */ + int ratelimitInterval; + int ratelimitBurst; + intTiny ratelimitSev; /* severity level (and below) for which rate-limiting shall apply */ + struct hashtable *ht; /* our hashtable for rate-limiting */ + sbool bParseHost; /* should parser parse host name? read-only after startup */ + sbool bCreatePath; /* auto-creation of socket directory? */ + sbool bUseCreds; /* pull original creator credentials from socket */ + sbool bWritePid; /* write original PID into tag */ } lstn_t; static lstn_t listeners[MAXFUNIX]; @@ -142,15 +145,20 @@ static int bOmitLocalLogging = 0; static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ -static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ -static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ -static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bWritePid = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bWritePidSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ #define DFLT_ratelimitInterval 2 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ +static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 static int ratelimitBurst = DFLT_ratelimitBurst; /* max nbr of messages in interval */ +static int ratelimitBurstSysSock = DFLT_ratelimitBurst; /* max nbr of messages in interval */ +#define DFLT_ratelimitSeverity 1 /* do not rate-limit emergency messages */ +static int ratelimitSeverity = DFLT_ratelimitSeverity; +static int ratelimitSeveritySysSock = DFLT_ratelimitSeverity; @@ -192,7 +200,7 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) if(rs->missed) { snprintf((char*)msgbuf, sizeof(msgbuf), "imuxsock lost %u messages due to rate-limiting", rs->missed); - logmsgInternal(0, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); rs->missed = 0; } rs->begin = 0; @@ -262,11 +270,23 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) pLogHostName = NULL; } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); + if(ratelimitInterval > 0) { + if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + /* in this case, we simply turn of rate-limiting */ + dbgprintf("imuxsock: turning off rate limiting because we could not " + "create hash table\n"); + ratelimitInterval = 0; + } + } + listeners[nfd].ratelimitInterval = ratelimitInterval; + listeners[nfd].ratelimitBurst = ratelimitBurst; + listeners[nfd].ratelimitSev = ratelimitSeverity; listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; listeners[nfd].bCreatePath = bCreatePath; listeners[nfd].sockName = pNewVal; - listeners[nfd].bUseCreds = bUseCreds; + listeners[nfd].bUseCreds = (bWritePid || ratelimitInterval) ? 1 : 0; + listeners[nfd].bWritePid = bWritePid; nfd++; } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", @@ -278,11 +298,10 @@ finalize_it: } -/* free the sockName[] socket names - needed as cleanup on several places - * note that nfd is NOT reset! sockName[0] is never freed, as it comes from +/* discard all log sockets except for "socket" 0. Data for it comes from * the constant memory pool - and if not, it is freeed via some other pointer. */ -static rsRetVal discardFunixn(void) +static rsRetVal discardLogSockets(void) { int i; @@ -294,6 +313,9 @@ static rsRetVal discardFunixn(void) if(listeners[i].hostName != NULL) { prop.Destruct(&(listeners[i].hostName)); } + if(listeners[i].ht != NULL) { + hashtable_destroy(listeners[i].ht, 1); /* 1 => free all values automatically */ + } } return RS_RET_OK; @@ -410,7 +432,7 @@ finalize_it: * Returns NULL if not found. */ static inline rsRetVal -findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) +findRatelimiter(lstn_t *pLstn, struct ucred *cred, rs_ratelimit_state_t **prl) { rs_ratelimit_state_t *rl; int r; @@ -420,7 +442,7 @@ findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) if(cred == NULL) FINALIZE; - rl = hashtable_search(hashtab, &cred->pid); + rl = hashtable_search(pLstn->ht, &cred->pid); if(rl == NULL) { /* we need to add a new ratelimiter, process not seen before! */ dbgprintf("imuxsock: no ratelimiter for pid %lu, creating one\n", @@ -429,8 +451,8 @@ findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) CHKmalloc(rl = malloc(sizeof(rs_ratelimit_state_t))); CHKmalloc(keybuf = malloc(sizeof(pid_t))); *keybuf = cred->pid; - initRatelimitState(rl, ratelimitInterval, ratelimitBurst); - r = hashtable_insert(hashtab, keybuf, rl); + initRatelimitState(rl, ratelimitInterval, pLstn->ratelimitBurst); + r = hashtable_insert(pLstn->ht, keybuf, rl); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -482,13 +504,33 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) int i; uchar *parse; int pri; + int facil; + int sever; uchar bufParseTAG[CONF_TAG_MAXSIZE]; struct syslogTime st; time_t tt; - rs_ratelimit_state_t *ratelimiter; + rs_ratelimit_state_t *ratelimiter = NULL; DEFiRet; - findRatelimiter(cred, &ratelimiter); /* ignore error, better so than others... */ +// TODO: handle format errors?? + /* we need to parse the pri first, because we need the severity for + * rate-limiting as well. + */ + parse = pRcv; + lenMsg = lenRcv; + + parse++; lenMsg--; /* '<' */ + pri = 0; + while(lenMsg && isdigit(*parse)) { + pri = pri * 10 + *parse - '0'; + ++parse; + --lenMsg; + } + facil = LOG_FAC(pri); + sever = LOG_PRI(pri); + + if(sever >= pLstn->ratelimitSev) + findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ datetime.getCurrTime(&st, &tt); if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { @@ -502,26 +544,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); - -// TODO: handle format errors - parse = pRcv; - lenMsg = lenRcv; - - parse++; lenMsg--; /* '<' */ - pri = 0; - while(lenMsg && isdigit(*parse)) { - pri = pri * 10 + *parse - '0'; - ++parse; - --lenMsg; - } - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); + pMsg->iFacility = facil; + pMsg->iSeverity = sever; MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); parse++; lenMsg--; /* '>' */ if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { - dbgprintf("we have a problem, invalid timestamp in msg!\n"); + DBGPRINTF("we have a problem, invalid timestamp in msg!\n"); } /* pull tag */ @@ -532,7 +562,8 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) --lenMsg; } bufParseTAG[i] = '\0'; /* terminate string */ - fixPID(bufParseTAG, &i, cred); + if(pLstn->bWritePid) + fixPID(bufParseTAG, &i, cred); MsgSetTAG(pMsg, bufParseTAG, i); MsgSetMSGoffs(pMsg, lenRcv - lenMsg); @@ -716,7 +747,19 @@ CODESTARTwillRun # endif if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; - listeners[0].bUseCreds = bUseCredsSysSock; + if(ratelimitIntervalSysSock > 0) { + if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + /* in this case, we simply turn of rate-limiting */ + dbgprintf("imuxsock: turning off rate limiting because we could not " + "create hash table\n"); + ratelimitIntervalSysSock = 0; + } + } + listeners[0].ratelimitInterval = ratelimitIntervalSysSock; + listeners[0].ratelimitBurst = ratelimitBurstSysSock; + listeners[0].ratelimitSev = ratelimitSeveritySysSock; + listeners[0].bUseCreds = (bWritePidSysSock || ratelimitIntervalSysSock) ? 1 : 0; + listeners[0].bWritePid = bWritePidSysSock; /* initialize and return if will run or not */ actSocks = 0; @@ -724,7 +767,7 @@ CODESTARTwillRun if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); - } + } } if(actSocks == 0) { @@ -737,8 +780,6 @@ CODESTARTwillRun CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); - CHKmalloc(hashtab = create_hashtable(1000, hash_from_key_fn, key_equals_fn)); - finalize_it: ENDwillRun @@ -765,16 +806,12 @@ CODESTARTafterRun free(pLogSockName); free(pLogHostName); - discardFunixn(); + discardLogSockets(); nfd = 1; if(pInputName != NULL) prop.Destruct(&pInputName); - if(hashtab != NULL) { - hashtable_destroy(hashtab, 1); /* 1 => free all values automatically */ - hashtab = NULL; - } ENDafterRun @@ -815,15 +852,19 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a pLogHostName = NULL; } - discardFunixn(); + discardLogSockets(); nfd = 1; bIgnoreTimestamp = 1; bUseFlowCtl = 0; - bUseCreds = 0; - bUseCredsSysSock = 0; + bWritePid = 0; + bWritePidSysSock = 0; bCreatePath = DFLT_bCreatePath; ratelimitInterval = DFLT_ratelimitInterval; + ratelimitIntervalSysSock = DFLT_ratelimitInterval; ratelimitBurst = DFLT_ratelimitBurst; + ratelimitBurstSysSock = DFLT_ratelimitBurst; + ratelimitSeverity = DFLT_ratelimitSeverity; + ratelimitSeveritySysSock = DFLT_ratelimitSeverity; return RS_RET_OK; } @@ -881,13 +922,15 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); + NULL, &bWritePid, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitinterval", 0, eCmdHdlrInt, NULL, &ratelimitInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitburst", 0, eCmdHdlrInt, NULL, &ratelimitBurst, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitseverity", 0, eCmdHdlrInt, + NULL, &ratelimitSeverity, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); /* the following one is a (dirty) trick: the system log socket is not added via @@ -901,7 +944,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); + NULL, &bWritePidSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitinterval", 0, eCmdHdlrInt, + NULL, &ratelimitIntervalSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitburst", 0, eCmdHdlrInt, + NULL, &ratelimitBurstSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitseverity", 0, eCmdHdlrInt, + NULL, &ratelimitSeveritySysSock, STD_LOADABLE_MODULE_ID)); /* support statistics gathering */ CHKiRet(statsobj.Construct(&modStats)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index c58c259b..43203378 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -457,6 +457,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_EPOLL_CTL_FAILED = -2174, /**< epoll_ctl() failed */ RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ + RS_RET_RATE_LIMITED = -2177, /**< some messages discarded due to exceeding a rate limit */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/tests/syslog_caller.c b/tests/syslog_caller.c new file mode 100644 index 00000000..91a1f08b --- /dev/null +++ b/tests/syslog_caller.c @@ -0,0 +1,20 @@ +#include +#include +#include + +int main(int argc, char *argv[]) +{ + int i; + int sev = 0; + if(argc != 2) { + fprintf(stderr, "usage: syslog_caller num-messages\n"); + exit(1); + } + + int msgs = atoi(argv[1]); + + for(i = 0 ; i < msgs ; ++i) { + syslog(sev % 8, "test message nbr %d, severity=%d", i, sev % 8); + sev++; + } +} -- cgit v1.2.3 From f4d6418102033c22b0ae71d496e993b00773b5c4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 17:52:15 +0200 Subject: doc/imuxsock: added new options to doc --- ChangeLog | 14 +++++++++++++- doc/imuxsock.html | 38 ++++++++++++++++++++++++++++++++++++-- plugins/imuxsock/imuxsock.c | 2 +- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca1428b4..f9b79392 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,11 +1,23 @@ --------------------------------------------------------------------------- Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log - socket itself (thanks to Lennart Poettering for the suggestion) + socket itself + (thanks to Lennart Poettering for the suggesting this feature) +- imuxsock now optionally uses per-process input rate limiting, guarding the + user against processes spamming the system log + (thanks to Lennart Poettering for suggesting this feature) - added new config statements * $InputUnixListenSocketUsePIDFromSystem * $SystemLogUsePIDFromSystem + * $SystemLogRateLimitInterval + * $SystemLogRateLimitBurst + * $SystemLogRateLimitSeverity + * $IMUxSockRateLimitInterval + * $IMUxSockRateLimitBurst + * $IMUxSockRateLimitSeverity - imuxsock now supports up to 50 different sockets for input +- some code cleanup in imuxsock (consider this a release a major + modification, especially if problems show up) --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 4af2c030..f3ee7be2 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -25,6 +25,19 @@ the past four years. Alternate behaviour may be desirable if gateway-like processes send messages via the local log slot - in this case, it can be enabled via the $InputUnixListenSocketIgnoreMsgTimestamp and $SystemLogSocketIgnoreMsgTimestamp config directives

    +

    There is input rate limiting available, (since 5.7.1) to guard you against +the problems of a wild running logging process. +If more than $IMUXSockRateLimitInterval * $IMUXSockRateLimitBurst log messages are emitted +from the same process, those messages with $IMUXSockRateLimitSeverity or lower will be +dropped. It is not possible to recover anything about these messages, but imuxsock will +tell you how many it has dropped one the interval has expired AND the next message +is logged. Rate-limiting depends on SCM_CREDENTIALS. If the platform does not support +this socket option, rate limiting is turned off. If multiple sockets are configured, +rate limiting works independently on each of them (that should be what you usually expect). +The same functionality is available for the system log socket, which +just uses the prefix $SystemLogRateLimit... but otherwise works exactly the same. +When working with severities, please keep in mind that higher severity numbers mean lower +severity and configure things accordingly.

    Unix log sockets can be flow-controlled. That is, if processing queues fill up, the unix socket reader is blocked for a short while. This may be useful to prevent overruning the queues (which may cause exessive disk-io where it actually would not be needed). However, @@ -40,6 +53,15 @@ the implications. Note that for many systems, turning on flow control does not h
    Ignore timestamps included in the message. Applies to the next socket being added.

  • $InputUnixListenSocketFlowControl [on/off] - specifies if flow control should be applied to the next socket.
    • +
  • $IMUXSockRateLimitInterval [number] - specifies the rate-limiting +interval in seconds. Default value is 5 seconds. +
    • +
  • $IMUXSockRateLimitBurst [number] - specifies the rate-limiting +burst in number of messages. Default is 200. +
    • +
  • $IMUXSockRateLimitSeverity [numerical severity] - specifies the severity of +messages that shall be rate-limited. +
  • $InputUnixListenSocketUsePIDFromSystem [on/off] - specifies if the pid being logged shall be obtained from the log socket itself. If so, the TAG part of the message is rewritten. It is recommended to turn this option on, but the default is "off" to keep compatible @@ -54,6 +76,15 @@ to the system log socket.
    • be obtained from the log socket itself. If so, the TAG part of the message is rewritten. It is recommended to turn this option on, but the default is "off" to keep compatible with earlier versions of rsyslog. This option was introduced in 5.7.0. +
  • $SystemLogRateLimitInterval [number] - specifies the rate-limiting +interval in seconds. Default value is 5 seconds. +
    • +
  • $SystemLogRateLimitBurst [number] - specifies the rate-limiting +burst in number of messages. Default is 200. +
    • +
  • $SystemLogRateLimitSeverity [numerical severity] - specifies the severity of +messages that shall be rate-limited. +
  • $InputUnixListenSocketCreatePath [on/off] - create directories in the socket path if they do not already exist. They are created with 0755 permissions with the owner being the process under which rsyslogd runs. The default is not to create directories. Keep in mind, though, that rsyslogd always @@ -74,8 +105,11 @@ will only affect the next one and then automatically be reset. This functionalit that the local hostname can be overridden in cases where that is desired.
    • Caveats/Known Bugs:
    -
    -This documentation is sparse and incomplete. +
      +
    • There is a compile-time limit of 50 concurrent sockets. If you need more, you need to +change the array size in imuxsock.c. +
    • This documentation is sparse and incomplete. +

    Sample:

    The following sample is the minimum setup required to accept syslog messages from applications running on the local system.
    diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index d500fc54..b4ee8db0 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -150,7 +150,7 @@ static int bWritePid = 0; /* use credentials from recvmsg() and fixup PID in TA static int bWritePidSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ -#define DFLT_ratelimitInterval 2 +#define DFLT_ratelimitInterval 5 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 -- cgit v1.2.3 From c7e36e6b7e3e9fb24dfd82e2cb7683bfee0c15ad Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 18:12:40 +0200 Subject: build system: fixed some minor issues --- configure.ac | 2 +- runtime/Makefile.am | 2 + runtime/hashtable.h | 199 ++++++++++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.h | 199 ------------------------------------------ 4 files changed, 202 insertions(+), 200 deletions(-) create mode 100644 runtime/hashtable.h delete mode 100644 runtime/hashtable/hashtable.h diff --git a/configure.ac b/configure.ac index c52749c8..1050981b 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.7.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.7.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/runtime/Makefile.am b/runtime/Makefile.am index d657b6ca..5a0c4437 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -96,6 +96,8 @@ librsyslog_la_SOURCES = \ ../parse.h \ \ hashtable/hashtable.c \ + hashtable.h \ + hashtable/hashtable_private.h \ \ ../outchannel.c \ ../outchannel.h \ diff --git a/runtime/hashtable.h b/runtime/hashtable.h new file mode 100644 index 00000000..b90781ab --- /dev/null +++ b/runtime/hashtable.h @@ -0,0 +1,199 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_H__ +#define __HASHTABLE_CWC22_H__ + +struct hashtable; + +/* Example of use: + * + * struct hashtable *h; + * struct some_key *k; + * struct some_value *v; + * + * static unsigned int hash_from_key_fn( void *k ); + * static int keys_equal_fn ( void *key1, void *key2 ); + * + * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); + * k = (struct some_key *) malloc(sizeof(struct some_key)); + * v = (struct some_value *) malloc(sizeof(struct some_value)); + * + * (initialise k and v to suitable values) + * + * if (! hashtable_insert(h,k,v) ) + * { exit(-1); } + * + * if (NULL == (found = hashtable_search(h,k) )) + * { printf("not found!"); } + * + * if (NULL == (found = hashtable_remove(h,k) )) + * { printf("Not found\n"); } + * + */ + +/* Macros may be used to define type-safe(r) hashtable access functions, with + * methods specialized to take known key and value types as parameters. + * + * Example: + * + * Insert this at the start of your file: + * + * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); + * + * This defines the functions 'insert_some', 'search_some' and 'remove_some'. + * These operate just like hashtable_insert etc., with the same parameters, + * but their function signatures have 'struct some_key *' rather than + * 'void *', and hence can generate compile time errors if your program is + * supplying incorrect data as a key (and similarly for value). + * + * Note that the hash and key equality functions passed to create_hashtable + * still take 'void *' parameters instead of 'some key *'. This shouldn't be + * a difficult issue as they're only defined and passed once, and the other + * functions will ensure that only valid keys are supplied to them. + * + * The cost for this checking is increased code size and runtime overhead + * - if performance is important, it may be worth switching back to the + * unsafe methods once your program has been debugged with the safe methods. + * This just requires switching to some simple alternative defines - eg: + * #define insert_some hashtable_insert + * + */ + +/***************************************************************************** + * create_hashtable + + * @name create_hashtable + * @param minsize minimum initial size of hashtable + * @param hashfunction function for hashing keys + * @param key_eq_fn function for determining key equality + * @return newly created hashtable or NULL on failure + */ + +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashfunction) (void*), + int (*key_eq_fn) (void*,void*)); + +/***************************************************************************** + * hashtable_insert + + * @name hashtable_insert + * @param h the hashtable to insert into + * @param k the key - hashtable claims ownership and will free on removal + * @param v the value - does not claim ownership + * @return non-zero for successful insertion + * + * This function will cause the table to expand if the insertion would take + * the ratio of entries to table size over the maximum load factor. + * + * This function does not check for repeated insertions with a duplicate key. + * The value returned when using a duplicate key is undefined -- when + * the hashtable changes size, the order of retrieval of duplicate key + * entries is reversed. + * If in doubt, remove before insert. + */ + +int +hashtable_insert(struct hashtable *h, void *k, void *v); + +#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ +int fnname (struct hashtable *h, keytype *k, valuetype *v) \ +{ \ + return hashtable_insert(h,k,v); \ +} + +/***************************************************************************** + * hashtable_search + + * @name hashtable_search + * @param h the hashtable to search + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * +hashtable_search(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_search(h,k)); \ +} + +/***************************************************************************** + * hashtable_remove + + * @name hashtable_remove + * @param h the hashtable to remove the item from + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * /* returns value */ +hashtable_remove(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_remove(h,k)); \ +} + + +/***************************************************************************** + * hashtable_count + + * @name hashtable_count + * @param h the hashtable + * @return the number of items stored in the hashtable + */ +unsigned int +hashtable_count(struct hashtable *h); + + +/***************************************************************************** + * hashtable_destroy + + * @name hashtable_destroy + * @param h the hashtable + * @param free_values whether to call 'free' on the remaining values + */ + +void +hashtable_destroy(struct hashtable *h, int free_values); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.h b/runtime/hashtable/hashtable.h deleted file mode 100644 index b90781ab..00000000 --- a/runtime/hashtable/hashtable.h +++ /dev/null @@ -1,199 +0,0 @@ -/* Copyright (C) 2002 Christopher Clark */ - -#ifndef __HASHTABLE_CWC22_H__ -#define __HASHTABLE_CWC22_H__ - -struct hashtable; - -/* Example of use: - * - * struct hashtable *h; - * struct some_key *k; - * struct some_value *v; - * - * static unsigned int hash_from_key_fn( void *k ); - * static int keys_equal_fn ( void *key1, void *key2 ); - * - * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); - * k = (struct some_key *) malloc(sizeof(struct some_key)); - * v = (struct some_value *) malloc(sizeof(struct some_value)); - * - * (initialise k and v to suitable values) - * - * if (! hashtable_insert(h,k,v) ) - * { exit(-1); } - * - * if (NULL == (found = hashtable_search(h,k) )) - * { printf("not found!"); } - * - * if (NULL == (found = hashtable_remove(h,k) )) - * { printf("Not found\n"); } - * - */ - -/* Macros may be used to define type-safe(r) hashtable access functions, with - * methods specialized to take known key and value types as parameters. - * - * Example: - * - * Insert this at the start of your file: - * - * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); - * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); - * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); - * - * This defines the functions 'insert_some', 'search_some' and 'remove_some'. - * These operate just like hashtable_insert etc., with the same parameters, - * but their function signatures have 'struct some_key *' rather than - * 'void *', and hence can generate compile time errors if your program is - * supplying incorrect data as a key (and similarly for value). - * - * Note that the hash and key equality functions passed to create_hashtable - * still take 'void *' parameters instead of 'some key *'. This shouldn't be - * a difficult issue as they're only defined and passed once, and the other - * functions will ensure that only valid keys are supplied to them. - * - * The cost for this checking is increased code size and runtime overhead - * - if performance is important, it may be worth switching back to the - * unsafe methods once your program has been debugged with the safe methods. - * This just requires switching to some simple alternative defines - eg: - * #define insert_some hashtable_insert - * - */ - -/***************************************************************************** - * create_hashtable - - * @name create_hashtable - * @param minsize minimum initial size of hashtable - * @param hashfunction function for hashing keys - * @param key_eq_fn function for determining key equality - * @return newly created hashtable or NULL on failure - */ - -struct hashtable * -create_hashtable(unsigned int minsize, - unsigned int (*hashfunction) (void*), - int (*key_eq_fn) (void*,void*)); - -/***************************************************************************** - * hashtable_insert - - * @name hashtable_insert - * @param h the hashtable to insert into - * @param k the key - hashtable claims ownership and will free on removal - * @param v the value - does not claim ownership - * @return non-zero for successful insertion - * - * This function will cause the table to expand if the insertion would take - * the ratio of entries to table size over the maximum load factor. - * - * This function does not check for repeated insertions with a duplicate key. - * The value returned when using a duplicate key is undefined -- when - * the hashtable changes size, the order of retrieval of duplicate key - * entries is reversed. - * If in doubt, remove before insert. - */ - -int -hashtable_insert(struct hashtable *h, void *k, void *v); - -#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ -int fnname (struct hashtable *h, keytype *k, valuetype *v) \ -{ \ - return hashtable_insert(h,k,v); \ -} - -/***************************************************************************** - * hashtable_search - - * @name hashtable_search - * @param h the hashtable to search - * @param k the key to search for - does not claim ownership - * @return the value associated with the key, or NULL if none found - */ - -void * -hashtable_search(struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ -valuetype * fnname (struct hashtable *h, keytype *k) \ -{ \ - return (valuetype *) (hashtable_search(h,k)); \ -} - -/***************************************************************************** - * hashtable_remove - - * @name hashtable_remove - * @param h the hashtable to remove the item from - * @param k the key to search for - does not claim ownership - * @return the value associated with the key, or NULL if none found - */ - -void * /* returns value */ -hashtable_remove(struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ -valuetype * fnname (struct hashtable *h, keytype *k) \ -{ \ - return (valuetype *) (hashtable_remove(h,k)); \ -} - - -/***************************************************************************** - * hashtable_count - - * @name hashtable_count - * @param h the hashtable - * @return the number of items stored in the hashtable - */ -unsigned int -hashtable_count(struct hashtable *h); - - -/***************************************************************************** - * hashtable_destroy - - * @name hashtable_destroy - * @param h the hashtable - * @param free_values whether to call 'free' on the remaining values - */ - -void -hashtable_destroy(struct hashtable *h, int free_values); - -#endif /* __HASHTABLE_CWC22_H__ */ - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -- cgit v1.2.3 From 829a446c9f00e2f7016f09e18358458e2cc02189 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 29 Sep 2010 12:44:04 +0200 Subject: improved imuxsock doc & added small testing tool permanently --- doc/imuxsock.html | 9 +++++---- tests/Makefile.am | 5 ++++- tests/syslog_caller.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/doc/imuxsock.html b/doc/imuxsock.html index f3ee7be2..1ab99a76 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -27,15 +27,16 @@ case, it can be enabled via the $InputUnixListenSocketIgnoreMsgTimestamp and $SystemLogSocketIgnoreMsgTimestamp config directives

    There is input rate limiting available, (since 5.7.1) to guard you against the problems of a wild running logging process. -If more than $IMUXSockRateLimitInterval * $IMUXSockRateLimitBurst log messages are emitted -from the same process, those messages with $IMUXSockRateLimitSeverity or lower will be +If more than $SystemLogRateLimitInterval * $SystemLogRateLimitBurst log messages are emitted +from the same process, those messages with $SystemLogRateLimitSeverity or lower will be dropped. It is not possible to recover anything about these messages, but imuxsock will tell you how many it has dropped one the interval has expired AND the next message is logged. Rate-limiting depends on SCM_CREDENTIALS. If the platform does not support this socket option, rate limiting is turned off. If multiple sockets are configured, rate limiting works independently on each of them (that should be what you usually expect). -The same functionality is available for the system log socket, which -just uses the prefix $SystemLogRateLimit... but otherwise works exactly the same. +The same functionality is available for additional log sockets, in which case the +config statements just use +the prefix $IMUXSockRateLimit... but otherwise works exactly the same. When working with severities, please keep in mind that higher severity numbers mean lower severity and configure things accordingly.

    Unix log sockets can be flow-controlled. That is, if processing queues fill up, diff --git a/tests/Makefile.am b/tests/Makefile.am index 8538140e..62d45228 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller TESTS = $(TESTRUNS) cfg.sh \ arrayqueue.sh \ linkedlistqueue.sh \ @@ -322,6 +322,9 @@ chkseq_SOURCES = chkseq.c tcpflood_SOURCES = tcpflood.c tcpflood_LDADD = $(SOL_LIBS) +syslog_caller_SOURCES = syslog_caller.c +syslog_caller_LDADD = $(SOL_LIBS) + diagtalker_SOURCES = diagtalker.c diagtalker_LDADD = $(SOL_LIBS) diff --git a/tests/syslog_caller.c b/tests/syslog_caller.c index 91a1f08b..60f24166 100644 --- a/tests/syslog_caller.c +++ b/tests/syslog_caller.c @@ -1,3 +1,31 @@ +/* A very primitive testing tool that just emits a number of + * messages to the system log socket. Currently sufficient, but + * obviously room for improvement. + * + * It is highly suggested NOT to "base" any derivative work + * on this tool ;) + * + * Part of the testbench for rsyslog. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ #include #include #include -- cgit v1.2.3 From 33888b84e7dc6ec4341e066c68c7077cade24a7b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 29 Sep 2010 14:31:44 +0200 Subject: imuxsock: indicate when rate limiting begins, include pid in rl messages also improved test tool --- plugins/imuxsock/imuxsock.c | 13 ++++++++++--- tests/syslog_caller.c | 41 ++++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index b4ee8db0..24bcebb7 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -180,7 +180,7 @@ initRatelimitState(struct rs_ratelimit_state *rs, unsigned short interval, unsig * be called concurrently. */ static inline int -withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) +withinRatelimit(struct rs_ratelimit_state *rs, time_t tt, pid_t pid) { int ret; uchar msgbuf[1024]; @@ -199,7 +199,8 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) if(tt > rs->begin + rs->interval) { if(rs->missed) { snprintf((char*)msgbuf, sizeof(msgbuf), - "imuxsock lost %u messages due to rate-limiting", rs->missed); + "imuxsock lost %u messages from pid %lu due to rate-limiting", + rs->missed, (unsigned long) pid); logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); rs->missed = 0; } @@ -212,6 +213,12 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) rs->done++; ret = 1; } else { + if(rs->missed == 0) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "imuxsock begins to drop messages from pid %lu due to rate-limiting", + (unsigned long) pid); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + } rs->missed++; ret = 0; } @@ -533,7 +540,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ datetime.getCurrTime(&st, &tt); - if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { + if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt, cred->pid)) { STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); FINALIZE; } diff --git a/tests/syslog_caller.c b/tests/syslog_caller.c index 60f24166..3f2702a6 100644 --- a/tests/syslog_caller.c +++ b/tests/syslog_caller.c @@ -5,6 +5,11 @@ * It is highly suggested NOT to "base" any derivative work * on this tool ;) * + * Options + * + * -s severity (0..7 accoding to syslog spec, r "rolling", default 6) + * -m number of messages to generate (default 500) + * * Part of the testbench for rsyslog. * * Copyright 2010 Rainer Gerhards and Adiscon GmbH. @@ -28,21 +33,43 @@ */ #include #include +#include #include +static void usage(void) +{ + fprintf(stderr, "usage: syslog_caller num-messages\n"); + exit(1); +} + + int main(int argc, char *argv[]) { int i; - int sev = 0; - if(argc != 2) { - fprintf(stderr, "usage: syslog_caller num-messages\n"); - exit(1); - } + int opt; + int bRollingSev = 0; + int sev = 6; + int msgs = 500; - int msgs = atoi(argv[1]); + while((opt = getopt(argc, argv, "m:s:")) != -1) { + switch (opt) { + case 's': if(*optarg == 'r') { + bRollingSev = 1; + sev = 0; + } else + sev = atoi(optarg); + break; + case 'm': msgs = atoi(optarg); + break; + default: usage(); + break; + } + } for(i = 0 ; i < msgs ; ++i) { syslog(sev % 8, "test message nbr %d, severity=%d", i, sev % 8); - sev++; + if(bRollingSev) + sev++; } + return(0); } -- cgit v1.2.3 From 11617d8ab5192c12e3b33cd9c08ac32f1d334a85 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 30 Sep 2010 14:17:34 +0200 Subject: omhdfs: first shot at this new module (very rough PoC code) --- Makefile.am | 4 + configure.ac | 19 +++ plugins/omhdfs/Makefile.am | 6 + plugins/omhdfs/omhdfs.c | 296 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 plugins/omhdfs/Makefile.am create mode 100644 plugins/omhdfs/omhdfs.c diff --git a/Makefile.am b/Makefile.am index bc6d8dd0..c7646e3d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -127,6 +127,10 @@ if ENABLE_OMUXSOCK SUBDIRS += plugins/omuxsock endif +if ENABLE_OMHDFS +SUBDIRS += plugins/omhdfs +endif + if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif diff --git a/configure.ac b/configure.ac index 1050981b..069a8fd8 100644 --- a/configure.ac +++ b/configure.ac @@ -1003,6 +1003,23 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to search for omtemplate to find everything! +# settings for the omhdfs; +AC_ARG_ENABLE(omhdfs, + [AS_HELP_STRING([--enable-omhdfs],[Compiles omhdfs template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omhdfs="yes" ;; + no) enable_omhdfs="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omhdfs) ;; + esac], + [enable_omhdfs=no] +) +# +# you may want to do some library checks here - see snmp, mysql, pgsql modules +# for samples +# +AM_CONDITIONAL(ENABLE_OMHDFS, test x$enable_omhdfs = xyes) + + AC_CONFIG_FILES([Makefile \ runtime/Makefile \ tools/Makefile \ @@ -1016,6 +1033,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ plugins/omtemplate/Makefile \ + plugins/omhdfs/Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/pmrfc3164sd/Makefile \ @@ -1070,6 +1088,7 @@ echo "---{ output plugins }---" echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" +echo " omhdfs module will be compiled: $enable_omhdfs" echo " omruleset module will be compiled: $enable_omruleset" echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am new file mode 100644 index 00000000..329e6816 --- /dev/null +++ b/plugins/omhdfs/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = omhdfs.la + +omhdfs_la_SOURCES = omhdfs.c +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +omhdfs_la_LDFLAGS = -module -avoid-version +omhdfs_la_LIBADD = diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c new file mode 100644 index 00000000..42e5dc8a --- /dev/null +++ b/plugins/omhdfs/omhdfs.c @@ -0,0 +1,296 @@ +/* omhdfs.c + * This is the implementation of the build-in file output module. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +/* this is kind of a hack, if defined, it instructs omhdfs to use + * the regular (non-hdfs) file system calls. This eases development + * (and hopefully troubleshooting) especially in cases when no + * hdfs environment is available. + */ +#define USE_REGULAR_FS 1 + +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "conf.h" +#include "cfsysline.h" +#include "module-template.h" +#include "unicode-helper.h" +#ifndef USE_REGULAR_FS +# include "hdfs.h" +#endif + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* globals for default values */ +static uchar *fileName = NULL; +/* end globals for default values */ + +typedef struct _instanceData { + uchar *fileName; +# ifdef USE_REGULAR_FS + short fd; /* file descriptor for (current) file */ +# else + hdfsFS fs; + hdfsFile fd; +# endif +} instanceData; + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + printf("omhdfs: file:%s", pData->fileName); + if (pData->fd == -1) + printf(" (unused)"); +ENDdbgPrintInstInfo + + + +#if 0 +static void prepareFile(instanceData *pData, uchar *newFileName) +{ + if(access((char*)newFileName, F_OK) == 0) { + /* file already exists */ + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fCreateMode); + } else { + pData->fd = -1; + /* file does not exist, create it (and eventually parent directories */ + if(pData->bCreateDirs) { + /* we fist need to create parent dirs if they are missing + * We do not report any errors here ourselfs but let the code + * fall through to error handler below. + */ + if(makeFileParentDirs(newFileName, strlen((char*)newFileName), + pData->fDirCreateMode, pData->dirUID, + pData->dirGID, pData->bFailOnChown) != 0) { + return; /* we give up */ + } + } + /* no matter if we needed to create directories or not, we now try to create + * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) + */ + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fCreateMode); + if(pData->fd != -1) { + /* check and set uid/gid */ + if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { + /* we need to set owner/group */ + if(fchown(pData->fd, pData->fileUID, + pData->fileGID) != 0) { + if(pData->bFailOnChown) { + int eSave = errno; + close(pData->fd); + pData->fd = -1; + errno = eSave; + } + /* we will silently ignore the chown() failure + * if configured to do so. + */ + } + } + } + } +} +#endif + +static void prepareFile(instanceData *pData, uchar *newFileName) +{ +# if USE_REGULAR_FS + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); +# else + pData->fs = hdfsConnect("default", 0); + pData->fd = hdfsOpenFile(fs, newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + if(!pData->fd) { + dbgprintf(stderr, "Failed to open %s for writing!\n", newFileName); + // TODO: suspend/error report + } + +# endif +} + +static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +{ + size_t lenWrite; + DEFiRet; + +# if USE_REGULAR_FS + if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { + int e = errno; + dbgprintf("omhdfs write error!\n"); + + /* If the filesystem is filled up, just ignore + * it for now and continue writing when possible + */ + //if(pData->fileType == eTypeFILE && e == ENOSPC) + //return RS_RET_OK; + + //(void) close(pData->fd); + iRet = RS_RET_DISABLE_ACTION; + errno = e; + //logerror((char*) pData->f_fname); + } +# else + lenWrite = strlen(char*ppString[0]); + tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); + if(num_written_bytes != lenWrite) { + dbgprintf("Failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, + lenWrite, num_written_bytes); + // TODO: suspend/error report + } +# endif +else { dbgprintf("omhdfs has successfully written to file\n"); } + + RETiRet; +} + + +BEGINcreateInstance +CODESTARTcreateInstance + pData->fd = -1; +ENDcreateInstance + + +BEGINfreeInstance +CODESTARTfreeInstance +# ifdef USE_REGULAR_FS + if(pData->fd != -1) + close(pData->fd); +# else + hdfsCloseFile(pData->fs, pData->fd); +# endif +ENDfreeInstance + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + dbgprintf(" (%s)\n", pData->fileName); + iRet = writeFile(ppString, iMsgOpts, pData); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct + + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omhdfs:", sizeof(":omhdfs:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* rgerhards 2004-11-17: from now, we need to have different + * processing, because after the first comma, the template name + * to use is specified. So we need to scan for the first coma first + * and then look at the rest of the line. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); + //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); + + // TODO: check for NULL filename + CHKmalloc(pData->fileName = ustrdup(fileName)); + + prepareFile(pData, pData->fileName); + +#if 0 + if ( pData->fd < 0 ){ + pData->fd = -1; + dbgprintf("Error opening log file: %s\n", pData->f_fname); + logerror((char*) pData->f_fname); + } +#endif +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +/* Reset config variables for this module to default values. + * rgerhards, 2007-07-17 + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ +/* + fileUID = -1; + fileGID = -1; + dirUID = -1; + dirGID = -1; + bFailOnChown = 1; + iDynaFileCacheSize = 10; + fCreateMode = 0644; + fDirCreateMode = 0700; + bCreateDirs = 1; +*/ + return RS_RET_OK; +} + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit -- cgit v1.2.3 From 7ac7ad166b82034eea4a37c1937ca5ddd618ec45 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 30 Sep 2010 13:55:26 +0000 Subject: omhdfs: added "real" libhdfs code, now actually works on hdfs very crude implementation, but probably good enough to gather some early performance data and experience with the module. No real error handling done, if something breaks, the whole thing will be blown up ;) --- plugins/omhdfs/Makefile.am | 4 +-- plugins/omhdfs/omhdfs.c | 62 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 329e6816..23fcf75a 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -omhdfs_la_LDFLAGS = -module -avoid-version +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux +omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64 -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64/server -ljava -ljvm -lverify omhdfs_la_LIBADD = diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 42e5dc8a..fcdff6e5 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -28,7 +28,7 @@ * (and hopefully troubleshooting) especially in cases when no * hdfs environment is available. */ -#define USE_REGULAR_FS 1 +//#define USE_REGULAR_FS 1 #include "config.h" #include "rsyslog.h" @@ -62,6 +62,8 @@ DEF_OMOD_STATIC_DATA /* globals for default values */ static uchar *fileName = NULL; +static uchar *hdfsHost = NULL; +int hdfsPort = 0; /* end globals for default values */ typedef struct _instanceData { @@ -71,6 +73,8 @@ typedef struct _instanceData { # else hdfsFS fs; hdfsFile fd; + const char *hdfsHost; + tPort hdfsPort; # endif } instanceData; @@ -85,8 +89,8 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo printf("omhdfs: file:%s", pData->fileName); - if (pData->fd == -1) - printf(" (unused)"); + //if (pData->fd == -1) + //printf(" (unused)"); ENDdbgPrintInstInfo @@ -139,22 +143,41 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } #endif -static void prepareFile(instanceData *pData, uchar *newFileName) +static void +prepareFile(instanceData *pData, uchar *newFileName) { # if USE_REGULAR_FS pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); # else - pData->fs = hdfsConnect("default", 0); - pData->fd = hdfsOpenFile(fs, newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + dbgprintf("omhdfs: try to connect to host '%s' at port %d\n", + pData->hdfsHost, pData->hdfsPort); + pData->fs = hdfsConnect(pData->hdfsHost, pData->hdfsPort); + if(pData->fs == NULL) { + dbgprintf("omhdfs: error can not connect to hdfs\n"); + } + pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_APPEND, 0, 0, 0); + if(pData->fd == NULL) { + /* maybe the file does not exist, so we try to create it now. + * Note that we can not use hdfsExists() because of a deficit in + * it: https://issues.apache.org/jira/browse/HDFS-1154 + * As of my testing, libhdfs at least seems to return ENOENT if + * the file does not exist. + */ + if(errno == ENOENT) { + dbgprintf("omhdfs: ENOENT trying to append to '%s', now trying create\n", + newFileName); + pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + } + } if(!pData->fd) { - dbgprintf(stderr, "Failed to open %s for writing!\n", newFileName); + dbgprintf("omhdfs: failed to open %s for writing!\n", newFileName); // TODO: suspend/error report } # endif } -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal writeFile(uchar **ppString, instanceData *pData) { size_t lenWrite; DEFiRet; @@ -176,15 +199,14 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa //logerror((char*) pData->f_fname); } # else - lenWrite = strlen(char*ppString[0]); + lenWrite = strlen((char*) ppString[0]); tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); - if(num_written_bytes != lenWrite) { - dbgprintf("Failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, - lenWrite, num_written_bytes); + if((unsigned) num_written_bytes != lenWrite) { + dbgprintf("omhdfs: failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, + lenWrite, (unsigned long) num_written_bytes); // TODO: suspend/error report } # endif -else { dbgprintf("omhdfs has successfully written to file\n"); } RETiRet; } @@ -192,7 +214,7 @@ else { dbgprintf("omhdfs has successfully written to file\n"); } BEGINcreateInstance CODESTARTcreateInstance - pData->fd = -1; + //pData->fd = -1; ENDcreateInstance @@ -214,7 +236,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction dbgprintf(" (%s)\n", pData->fileName); - iRet = writeFile(ppString, iMsgOpts, pData); + iRet = writeFile(ppString, pData); ENDdoAction @@ -241,6 +263,12 @@ CODESTARTparseSelectorAct // TODO: check for NULL filename CHKmalloc(pData->fileName = ustrdup(fileName)); + if(hdfsHost == NULL) { + pData->hdfsHost = "default"; + } else { + CHKmalloc(pData->hdfsHost = strdup((char*)hdfsHost)); + } + pData->hdfsPort = hdfsPort; prepareFile(pData, pData->fileName); @@ -271,6 +299,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a fDirCreateMode = 0700; bCreateDirs = 1; */ + hdfsHost = NULL; + hdfsPort = 0; return RS_RET_OK; } @@ -291,6 +321,8 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr ENDmodInit -- cgit v1.2.3 From 1d609fc5ab9b5d1c89d5d7d8f321c4e8493b4437 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 10:27:44 +0200 Subject: omhdfs: some cleanup of build system but still pretty ugly. Any java folks out there to help clean it up? --- doc/omhdfs.html | 49 +++++++++++++++++++++++++++++++++++++++++++ doc/rsyslog_conf_modules.html | 1 + plugins/omhdfs/Makefile.am | 5 +++-- 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 doc/omhdfs.html diff --git a/doc/omhdfs.html b/doc/omhdfs.html new file mode 100644 index 00000000..827697b6 --- /dev/null +++ b/doc/omhdfs.html @@ -0,0 +1,49 @@ + +rsyslog output module for HDFS (omhdfs) +back + + +

    Unix sockets Output Module (omhdfs)

    +

    Module Name:    omhdfs

    +

    Available since:    5.7.2

    +

    Author: Rainer Gerhards <rgerhards@adiscon.com>

    +

    Description:

    +

    This module supports writing message into files on Hadoop's HDFS +file system. +

    Configuration Directives:

    +
      +
    • $...
      +option... +
      • +
    +Caveats/Known Bugs: +

    Building omhdfs is a challenge because we could not yet find out how +to integrate Java properly into the autotools build process. The issue is +that HDFS is written in Java and libhdfs uses JNI to talk to it. That requires +that various system-specific environment options and pathes be set correctly. At +this point, we leave this to the user. If someone know how to do it better, +please drop us a line! +

    In order to build, you need to set these environment variables BEFORE running +./configure: +

      +
    • JAVA_INCLUDES - must have all include pathes that are needed to build +JNI C programms, including the -I options necessary for gcc. An example is
      +# export JAVA_INCLUDES="-I/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include -I/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/include/linux" +
    • JAVA_LIBS - must have all library pathes that are needed to build +JNI C programms, including the -l/-L options necessary for gcc. An example is
      +# export export JAVA_LIBS="-L/usr/java/jdk1.6.0_21/jre/lib/amd64 -L/usr/java/jdk1.6.0_21/jre/lib/amd64/server -ljava -ljvm -lverify" + +
    +

    Sample:

    +

    +

    + +[manual index] [rsyslog site]

    +

    This documentation is part of the rsyslog +project.
    +Copyright © 2010 by Rainer Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

    + + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 85899954..74aa319c 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -66,6 +66,7 @@ permits rsyslog to alert folks by mail if something important happens
  • omoracle - output module for Oracle (native OCI interface)
  • omudpspoof - output module sending UDP syslog messages with a spoofed address
  • omuxsock - output module Unix domain sockets
    • +
  • omhdfs - output module for Hadoop's HDFS file system

    • Parser Modules

    diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 23fcf75a..2e7ef8ea 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,6 +1,7 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux -omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64 -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64/server -ljava -ljvm -lverify +#omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(JAVA_INCLUDES) +omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs $(JAVA_LIBS) omhdfs_la_LIBADD = -- cgit v1.2.3 From d8a1489f545179591abced679ba24831d85ca224 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 11:29:50 +0200 Subject: omhdfs: cleanup and lots of improvement now things look much better, also done some prep in order to support a file cache (we need this for multiple selectors writing to the same file). --- plugins/omhdfs/omhdfs.c | 247 +++++++++++++++++++++++++++++------------------- runtime/rsyslog.h | 3 + 2 files changed, 153 insertions(+), 97 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index fcdff6e5..71080585 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -23,13 +23,6 @@ * A copy of the GPL can be found in the file "COPYING" in this distribution. */ -/* this is kind of a hack, if defined, it instructs omhdfs to use - * the regular (non-hdfs) file system calls. This eases development - * (and hopefully troubleshooting) especially in cases when no - * hdfs environment is available. - */ -//#define USE_REGULAR_FS 1 - #include "config.h" #include "rsyslog.h" #include @@ -42,6 +35,8 @@ #include #include #include +#include +#include #include "syslogd-types.h" #include "srUtils.h" @@ -50,15 +45,14 @@ #include "cfsysline.h" #include "module-template.h" #include "unicode-helper.h" -#ifndef USE_REGULAR_FS -# include "hdfs.h" -#endif +#include "errmsg.h" MODULE_TYPE_OUTPUT /* internal structures */ DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) /* globals for default values */ static uchar *fileName = NULL; @@ -66,18 +60,23 @@ static uchar *hdfsHost = NULL; int hdfsPort = 0; /* end globals for default values */ -typedef struct _instanceData { - uchar *fileName; -# ifdef USE_REGULAR_FS - short fd; /* file descriptor for (current) file */ -# else +typedef struct { + uchar *name; hdfsFS fs; - hdfsFile fd; + hdfsFile fh; const char *hdfsHost; tPort hdfsPort; -# endif + int nUsers; + pthread_mutex_t mut; +} file_t; + + +typedef struct _instanceData { + file_t *pFile; } instanceData; +/* forward definitions (down here, need data types) */ +static inline rsRetVal fileClose(file_t *pFile); BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -88,9 +87,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - printf("omhdfs: file:%s", pData->fileName); - //if (pData->fd == -1) - //printf(" (unused)"); + printf("omhdfs: file:%s", pData->pFile->name); ENDdbgPrintInstInfo @@ -100,10 +97,10 @@ static void prepareFile(instanceData *pData, uchar *newFileName) { if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, pData->fCreateMode); } else { - pData->fd = -1; + pData->fh = -1; /* file does not exist, create it (and eventually parent directories */ if(pData->bCreateDirs) { /* we fist need to create parent dirs if they are missing @@ -119,18 +116,18 @@ static void prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, pData->fCreateMode); - if(pData->fd != -1) { + if(pData->fh != -1) { /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, + if(fchown(pData->fh, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; - close(pData->fd); - pData->fd = -1; + close(pData->fh); + pData->fh = -1; errno = eSave; } /* we will silently ignore the chown() failure @@ -143,20 +140,71 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } #endif -static void -prepareFile(instanceData *pData, uchar *newFileName) +/* ---BEGIN FILE OBJECT---------------------------------------------------- */ +/* This code handles the "file object". This is split from the actual + * instance data, because several instances may write into the same file. + * If so, we need to use a single object, and also synchronize their writes. + * So we keep the file object separately, and just stick a reference into + * the instance data. + */ + +static inline rsRetVal +fileObjConstruct(file_t **ppFile) +{ + file_t *pFile; + DEFiRet; + + CHKmalloc(pFile = malloc(sizeof(file_t))); + pFile->name = NULL; + pFile->hdfsHost = NULL; + pFile->fh = NULL; + pFile->nUsers = 0; + + *ppFile = pFile; +finalize_it: + RETiRet; +} + +static inline void +fileObjAddUser(file_t *pFile) +{ + /* init mutex only when second user is added */ + ++pFile->nUsers; + if(pFile->nUsers == 2) + pthread_mutex_init(&pFile->mut, NULL); +} + +static inline rsRetVal +fileObjDestruct(file_t **ppFile) +{ + file_t *pFile = *ppFile; + if(pFile->nUsers > 1) + pthread_mutex_destroy(&pFile->mut); + fileClose(pFile); + free(pFile->name); + free((char*)pFile->hdfsHost); + free(pFile->fh); + + return RS_RET_OK; +} + +static inline rsRetVal +fileOpen(file_t *pFile) { -# if USE_REGULAR_FS - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); -# else - dbgprintf("omhdfs: try to connect to host '%s' at port %d\n", - pData->hdfsHost, pData->hdfsPort); - pData->fs = hdfsConnect(pData->hdfsHost, pData->hdfsPort); - if(pData->fs == NULL) { - dbgprintf("omhdfs: error can not connect to hdfs\n"); + DEFiRet; + + assert(pFile->fh == NULL); + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + DBGPRINTF("omhdfs: try to connect to HDFS at host '%s', port %d\n", + pFile->hdfsHost, pFile->hdfsPort); + pFile->fs = hdfsConnect(pFile->hdfsHost, pFile->hdfsPort); + if(pFile->fs == NULL) { + DBGPRINTF("omhdfs: error can not connect to hdfs\n"); + ABORT_FINALIZE(RS_RET_SUSPENDED); } - pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_APPEND, 0, 0, 0); - if(pData->fd == NULL) { + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_APPEND, 0, 0, 0); + if(pFile->fh == NULL) { /* maybe the file does not exist, so we try to create it now. * Note that we can not use hdfsExists() because of a deficit in * it: https://issues.apache.org/jira/browse/HDFS-1154 @@ -164,68 +212,75 @@ prepareFile(instanceData *pData, uchar *newFileName) * the file does not exist. */ if(errno == ENOENT) { - dbgprintf("omhdfs: ENOENT trying to append to '%s', now trying create\n", - newFileName); - pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + DBGPRINTF("omhdfs: ENOENT trying to append to '%s', now trying create\n", + pFile->name); + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_CREAT, 0, 0, 0); } } - if(!pData->fd) { - dbgprintf("omhdfs: failed to open %s for writing!\n", newFileName); - // TODO: suspend/error report + if(pFile->fh == NULL) { + DBGPRINTF("omhdfs: failed to open %s for writing!\n", pFile->name); + ABORT_FINALIZE(RS_RET_SUSPENDED); } -# endif +finalize_it: + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); + RETiRet; } -static rsRetVal writeFile(uchar **ppString, instanceData *pData) + +static inline rsRetVal +fileWrite(file_t *pFile, uchar *buf) { size_t lenWrite; DEFiRet; -# if USE_REGULAR_FS - if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { - int e = errno; - dbgprintf("omhdfs write error!\n"); - - /* If the filesystem is filled up, just ignore - * it for now and continue writing when possible - */ - //if(pData->fileType == eTypeFILE && e == ENOSPC) - //return RS_RET_OK; - - //(void) close(pData->fd); - iRet = RS_RET_DISABLE_ACTION; - errno = e; - //logerror((char*) pData->f_fname); - } -# else - lenWrite = strlen((char*) ppString[0]); - tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); + assert(pFile->fh != NULL); + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + lenWrite = strlen((char*) buf); + tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { - dbgprintf("omhdfs: failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, - lenWrite, (unsigned long) num_written_bytes); - // TODO: suspend/error report + errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, "omhdfs: failed to write %s, expected %lu bytes, " + "written %lu\n", pFile->name, lenWrite, (unsigned long) num_written_bytes); + ABORT_FINALIZE(RS_RET_SUSPENDED); } -# endif +finalize_it: + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); RETiRet; } +static inline rsRetVal +fileClose(file_t *pFile) +{ + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + if(pFile->fh != NULL) { + hdfsCloseFile(pFile->fs, pFile->fh); + pFile->fh = NULL; + } + + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); + + return RS_RET_OK; +} + +/* ---END FILE OBJECT---------------------------------------------------- */ + + BEGINcreateInstance CODESTARTcreateInstance - //pData->fd = -1; + pData->pFile = NULL; ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance -# ifdef USE_REGULAR_FS - if(pData->fd != -1) - close(pData->fd); -# else - hdfsCloseFile(pData->fs, pData->fd); -# endif + fileObjDestruct(&pData->pFile); ENDfreeInstance @@ -235,8 +290,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - dbgprintf(" (%s)\n", pData->fileName); - iRet = writeFile(ppString, pData); + DBGPRINTF(" (%s)\n", pData->pFile->name); + iRet = fileWrite(pData->pFile, ppString[0]); ENDdoAction @@ -251,34 +306,29 @@ CODESTARTparseSelectorAct /* ok, if we reach this point, we have something for us */ p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ CHKiRet(createInstance(&pData)); - CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* rgerhards 2004-11-17: from now, we need to have different - * processing, because after the first comma, the template name - * to use is specified. So we need to scan for the first coma first - * and then look at the rest of the line. - */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); - // TODO: check for NULL filename - CHKmalloc(pData->fileName = ustrdup(fileName)); + if(fileName == NULL) { + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: no file name specified, can not continue"); + ABORT_FINALIZE(RS_RET_FILE_NOT_SPECIFIED); + } + + CHKiRet(fileObjConstruct(&pData->pFile)); if(hdfsHost == NULL) { - pData->hdfsHost = "default"; + CHKmalloc(pData->pFile->hdfsHost = strdup("default")); } else { - CHKmalloc(pData->hdfsHost = strdup((char*)hdfsHost)); + CHKmalloc(pData->pFile->hdfsHost = strdup((char*)hdfsHost)); } - pData->hdfsPort = hdfsPort; + pData->pFile->hdfsPort = hdfsPort; - prepareFile(pData, pData->fileName); + fileOpen(pData->pFile); -#if 0 - if ( pData->fd < 0 ){ - pData->fd = -1; - dbgprintf("Error opening log file: %s\n", pData->f_fname); - logerror((char*) pData->f_fname); + if(pData->pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pData->pFile->name); + iRet = RS_RET_SUSPENDED; } -#endif CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -307,6 +357,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINmodExit CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -320,6 +371,8 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 43203378..7ccc9cb8 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -458,6 +458,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ RS_RET_RATE_LIMITED = -2177, /**< some messages discarded due to exceeding a rate limit */ + RS_RET_ERR_HDFS_WRITE = -2178, /**< error writing to HDFS */ + RS_RET_ERR_HDFS_OPEN = -2179, /**< error during hdfsOpen (e.g. file does not exist) */ + RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 0ee524c391d017225049542fffe572d7de7d1512 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 09:55:21 +0000 Subject: omhdfs: fixed small bug ... that I could only see on my execution environment (I right now have two envs, a local one where I can compile, but not run and a remote one where I can do both, but this is a bit less convenient to use). --- plugins/omhdfs/omhdfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 71080585..5d28d5dd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -316,6 +316,7 @@ CODESTARTparseSelectorAct } CHKiRet(fileObjConstruct(&pData->pFile)); + CHKmalloc(pData->pFile->name = (uchar*)strdup((char*)fileName)); if(hdfsHost == NULL) { CHKmalloc(pData->pFile->hdfsHost = strdup("default")); } else { -- cgit v1.2.3 From 9696cdef34f5d033564138fb9d4afb87daa6b1be Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 12:32:01 +0200 Subject: omhdfs: files now kept inside a hashtable for use by multiple actions Note:compiles, but not yet tested --- plugins/omhdfs/omhdfs.c | 40 +++++++++++++++++++++++++++++----------- runtime/hashtable.h | 4 +++- runtime/hashtable/hashtable.c | 27 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 5d28d5dd..eaca90e4 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -46,6 +46,7 @@ #include "module-template.h" #include "unicode-helper.h" #include "errmsg.h" +#include "hashtable.h" MODULE_TYPE_OUTPUT @@ -54,6 +55,9 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +/* global data */ +static struct hashtable *files; /* holds all file objects that we know */ + /* globals for default values */ static uchar *fileName = NULL; static uchar *hdfsHost = NULL; @@ -172,6 +176,7 @@ fileObjAddUser(file_t *pFile) ++pFile->nUsers; if(pFile->nUsers == 2) pthread_mutex_init(&pFile->mut, NULL); + dbgprintf("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); } static inline rsRetVal @@ -296,6 +301,9 @@ ENDdoAction BEGINparseSelectorAct + file_t *pFile; + int r; + uchar *keybuf; CODESTARTparseSelectorAct /* first check if this config line is actually for us */ @@ -315,21 +323,28 @@ CODESTARTparseSelectorAct ABORT_FINALIZE(RS_RET_FILE_NOT_SPECIFIED); } - CHKiRet(fileObjConstruct(&pData->pFile)); - CHKmalloc(pData->pFile->name = (uchar*)strdup((char*)fileName)); - if(hdfsHost == NULL) { - CHKmalloc(pData->pFile->hdfsHost = strdup("default")); - } else { - CHKmalloc(pData->pFile->hdfsHost = strdup((char*)hdfsHost)); + pFile = hashtable_search(files, fileName); + if(pFile == NULL) { + /* we need a new file object, this one not seen before */ + CHKiRet(fileObjConstruct(&pFile)); + CHKmalloc(pFile->name = (uchar*)strdup((char*)fileName)); + CHKmalloc(keybuf = ustrdup(fileName)); + r = hashtable_insert(files, keybuf, pFile); + if(r == 0) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } - pData->pFile->hdfsPort = hdfsPort; + fileObjAddUser(pFile); - fileOpen(pData->pFile); - - if(pData->pFile->fh == NULL){ - errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pData->pFile->name); + CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); + pFile->hdfsPort = hdfsPort; + fileOpen(pFile); + if(pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); iRet = RS_RET_SUSPENDED; } + + pData->pFile = pFile; + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -359,6 +374,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINmodExit CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); + if(files != NULL) + hashtable_destroy(files, 1); /* 1 => free all values automatically */ ENDmodExit @@ -373,6 +390,7 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); diff --git a/runtime/hashtable.h b/runtime/hashtable.h index b90781ab..0f980127 100644 --- a/runtime/hashtable.h +++ b/runtime/hashtable.h @@ -196,4 +196,6 @@ hashtable_destroy(struct hashtable *h, int free_values); * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ + */ +unsigned int hash_from_string(void *k) ; +int key_equals_string(void *key1, void *key2); diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c index a10e3bc6..e2a2b3f4 100644 --- a/runtime/hashtable/hashtable.c +++ b/runtime/hashtable/hashtable.c @@ -241,6 +241,33 @@ hashtable_destroy(struct hashtable *h, int free_values) free(h); } +/* some generic hash functions */ + +/* one provided by Aaaron Wiebe based on perl's hashng algorithm + * (so probably pretty generic). Not for excessively large strings! + */ +unsigned int +hash_from_string(void *k) +{ + int len; + char *rkey = (char*) k; + unsigned hashval = 1; + + len = (int) strlen(rkey); + while (len--) + hashval = hashval * 33 + *rkey++; + + return hashval; +} + + +int +key_equals_string(void *key1, void *key2) +{ + return strcmp(key1, key2); +} + + /* * Copyright (c) 2002, Christopher Clark * All rights reserved. -- cgit v1.2.3 From 255895a58b3f2a54fecf971da700caf265b4e1f0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 13:59:48 +0000 Subject: omhdfs: more improvements finally this looks almost production ready for files where no directory path needs to be created --- plugins/imuxsock/imuxsock.c | 4 +-- plugins/omhdfs/Makefile.am | 3 +-- plugins/omhdfs/omhdfs.c | 47 ++++++++++++++++++++++++----------- runtime/hashtable.h | 3 ++- runtime/hashtable/hashtable.c | 17 ++++++++++--- runtime/hashtable/hashtable_private.h | 1 + 6 files changed, 53 insertions(+), 22 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 24bcebb7..ad2d61c8 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -278,7 +278,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); if(ratelimitInterval > 0) { - if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); @@ -755,7 +755,7 @@ CODESTARTwillRun if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; if(ratelimitIntervalSysSock > 0) { - if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 2e7ef8ea..95e6b102 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,7 +1,6 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -#omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(JAVA_INCLUDES) omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs $(JAVA_LIBS) -omhdfs_la_LIBADD = +omhdfs_la_LIBADD = $(RSRT_LIBS) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index eaca90e4..734c28cd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -61,6 +61,7 @@ static struct hashtable *files; /* holds all file objects that we know */ /* globals for default values */ static uchar *fileName = NULL; static uchar *hdfsHost = NULL; +static uchar *dfltTplName = NULL; /* default template name to use */ int hdfsPort = 0; /* end globals for default values */ @@ -176,10 +177,10 @@ fileObjAddUser(file_t *pFile) ++pFile->nUsers; if(pFile->nUsers == 2) pthread_mutex_init(&pFile->mut, NULL); - dbgprintf("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); + DBGPRINTF("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); } -static inline rsRetVal +static rsRetVal fileObjDestruct(file_t **ppFile) { file_t *pFile = *ppFile; @@ -193,6 +194,18 @@ fileObjDestruct(file_t **ppFile) return RS_RET_OK; } +/* this function is to be used as destructor for the + * hash table code. + */ +static void +fileObjDestruct4Hashtable(void *ptr) +{ + dbgprintf("omfile: fileObjDestruct4Hashtable called\n"); + file_t *pFile = (file_t*) ptr; + fileObjDestruct(&pFile); +} + + static inline rsRetVal fileOpen(file_t *pFile) { @@ -261,6 +274,8 @@ finalize_it: static inline rsRetVal fileClose(file_t *pFile) { + DEFiRet; + if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); if(pFile->fh != NULL) { @@ -271,7 +286,7 @@ fileClose(file_t *pFile) if(pFile->nUsers > 1) d_pthread_mutex_unlock(&pFile->mut); - return RS_RET_OK; + RETiRet; } /* ---END FILE OBJECT---------------------------------------------------- */ @@ -285,7 +300,8 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - fileObjDestruct(&pData->pFile); + if(pData->pFile != NULL) + fileObjDestruct(&pData->pFile); ENDfreeInstance @@ -315,8 +331,8 @@ CODESTARTparseSelectorAct p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ CHKiRet(createInstance(&pData)); CODE_STD_STRING_REQUESTparseSelectorAct(1) - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); - //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, + (dfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : dfltTplName)); if(fileName == NULL) { errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: no file name specified, can not continue"); @@ -327,17 +343,18 @@ CODESTARTparseSelectorAct if(pFile == NULL) { /* we need a new file object, this one not seen before */ CHKiRet(fileObjConstruct(&pFile)); - CHKmalloc(pFile->name = (uchar*)strdup((char*)fileName)); + CHKmalloc(pFile->name = fileName); CHKmalloc(keybuf = ustrdup(fileName)); + fileName = NULL; /* re-set, data passed to file object */ + CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); + pFile->hdfsPort = hdfsPort; + fileOpen(pFile); r = hashtable_insert(files, keybuf, pFile); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } fileObjAddUser(pFile); - CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); - pFile->hdfsPort = hdfsPort; - fileOpen(pFile); if(pFile->fh == NULL){ errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); iRet = RS_RET_SUSPENDED; @@ -390,11 +407,13 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); - CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string)); + CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string, + fileObjDestruct4Hashtable)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsdefaulttemplate", 0, eCmdHdlrGetWord, NULL, &dfltTplName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr ENDmodInit diff --git a/runtime/hashtable.h b/runtime/hashtable.h index 0f980127..f777ad0b 100644 --- a/runtime/hashtable.h +++ b/runtime/hashtable.h @@ -68,13 +68,14 @@ struct hashtable; * @param minsize minimum initial size of hashtable * @param hashfunction function for hashing keys * @param key_eq_fn function for determining key equality + * @param dest destructor for value entries (NULL -> use free()) * @return newly created hashtable or NULL on failure */ struct hashtable * create_hashtable(unsigned int minsize, unsigned int (*hashfunction) (void*), - int (*key_eq_fn) (void*,void*)); + int (*key_eq_fn) (void*,void*), void (*dest) (void*)); /***************************************************************************** * hashtable_insert diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c index e2a2b3f4..41fc60fe 100644 --- a/runtime/hashtable/hashtable.c +++ b/runtime/hashtable/hashtable.c @@ -29,7 +29,7 @@ const float max_load_factor = 0.65; struct hashtable * create_hashtable(unsigned int minsize, unsigned int (*hashf) (void*), - int (*eqf) (void*,void*)) + int (*eqf) (void*,void*), void (*dest)(void*)) { struct hashtable *h; unsigned int pindex, size = primes[0]; @@ -49,6 +49,7 @@ create_hashtable(unsigned int minsize, h->entrycount = 0; h->hashfn = hashf; h->eqfn = eqf; + h->dest = dest; h->loadlimit = (unsigned int) ceil(size * max_load_factor); return h; } @@ -225,7 +226,16 @@ hashtable_destroy(struct hashtable *h, int free_values) { e = table[i]; while (NULL != e) - { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } + { + f = e; + e = e->next; + freekey(f->k); + if(h->dest == NULL) + free(f->v); + else + h->dest(f->v); + free(f); + } } } else @@ -264,7 +274,8 @@ hash_from_string(void *k) int key_equals_string(void *key1, void *key2) { - return strcmp(key1, key2); + /* we must return true IF the keys are equal! */ + return !strcmp(key1, key2); } diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h index 3e95f600..10b82da4 100644 --- a/runtime/hashtable/hashtable_private.h +++ b/runtime/hashtable/hashtable_private.h @@ -21,6 +21,7 @@ struct hashtable { unsigned int primeindex; unsigned int (*hashfn) (void *k); int (*eqfn) (void *k1, void *k2); + void (*dest) (void *v); /* destructor for values, if NULL use free() */ }; /*****************************************************************************/ -- cgit v1.2.3 From 670e81c9a8275a5509efd71dae66d9a267ec1574 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 14:36:39 +0000 Subject: omhdfs: made action suspend/resume working --- plugins/omhdfs/omhdfs.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 734c28cd..4fbf2ef4 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -276,16 +276,19 @@ fileClose(file_t *pFile) { DEFiRet; + if(pFile->fh == NULL) + FINALIZE; + if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); - if(pFile->fh != NULL) { - hdfsCloseFile(pFile->fs, pFile->fh); - pFile->fh = NULL; - } + + hdfsCloseFile(pFile->fs, pFile->fh); + pFile->fh = NULL; if(pFile->nUsers > 1) d_pthread_mutex_unlock(&pFile->mut); +finalize_it: RETiRet; } @@ -307,12 +310,26 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + fileClose(pData->pFile); + fileOpen(pData->pFile); + if(pData->pFile->fh == NULL){ + dbgprintf("omhdfs: tried to resume file %s, but still no luck...\n", + pData->pFile->name); + iRet = RS_RET_SUSPENDED; + } ENDtryResume BEGINdoAction CODESTARTdoAction DBGPRINTF(" (%s)\n", pData->pFile->name); + if(pData->pFile->fh == NULL) { + fileOpen(pData->pFile); + if(pData->pFile->fh == NULL) { + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } iRet = fileWrite(pData->pFile, ppString[0]); +finalize_it: ENDdoAction @@ -349,17 +366,16 @@ CODESTARTparseSelectorAct CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); pFile->hdfsPort = hdfsPort; fileOpen(pFile); + if(pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - " + "retrying later", pFile->name); + iRet = RS_RET_SUSPENDED; + } r = hashtable_insert(files, keybuf, pFile); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } fileObjAddUser(pFile); - - if(pFile->fh == NULL){ - errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); - iRet = RS_RET_SUSPENDED; - } - pData->pFile = pFile; CODE_STD_FINALIZERparseSelectorAct -- cgit v1.2.3 From d5f16404f93d54afddebb9fb683469fc712d2335 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 17:26:14 +0200 Subject: omhdfs: added doc --- ChangeLog | 3 +++ doc/omhdfs.html | 28 ++++++++++++++++++++++++---- plugins/omhdfs/omhdfs.c | 2 +- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index f9b79392..8129f634 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-09-?? +- support for Hadoop's HDFS added (via omhdfs) +--------------------------------------------------------------------------- Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log socket itself diff --git a/doc/omhdfs.html b/doc/omhdfs.html index 827697b6..3849f167 100644 --- a/doc/omhdfs.html +++ b/doc/omhdfs.html @@ -12,8 +12,19 @@ file system.

    Configuration Directives:

      -
    • $...
      -option... +
    • $OMHDFSFileName [name]
      +The name of the file to which the output data shall be written. +
      • +
    • $OMHDFSHost [name]
      +Name or IP address of the HDFS host to connect to. +
      • +
    • $OMHDFSPort [name]
      +Port on which to connect to the HDFS host. +
      • +
    • $OMHDFSDefaultTemplate [name]
      +Default template to be used when none is specified. This saves the work of +specifying the same template ever and ever again. Of course, the default +template can be overwritten via the usual method.
    Caveats/Known Bugs: @@ -23,7 +34,8 @@ that HDFS is written in Java and libhdfs uses JNI to talk to it. That requires that various system-specific environment options and pathes be set correctly. At this point, we leave this to the user. If someone know how to do it better, please drop us a line! -

    In order to build, you need to set these environment variables BEFORE running +

      +
    • In order to build, you need to set these environment variables BEFORE running ./configure:
      • JAVA_INCLUDES - must have all include pathes that are needed to build @@ -32,12 +44,20 @@ JNI C programms, including the -I options necessary for gcc. An example is
      • JAVA_LIBS - must have all library pathes that are needed to build JNI C programms, including the -l/-L options necessary for gcc. An example is
        # export export JAVA_LIBS="-L/usr/java/jdk1.6.0_21/jre/lib/amd64 -L/usr/java/jdk1.6.0_21/jre/lib/amd64/server -ljava -ljvm -lverify" +
      +
    • As of HDFS architecture, you must make sure that all relevant environment +variables (the usual Java stuff and HADOOP's home directory) are properly set. +
    • As it looks, libhdfs makes Java throw exceptions to stdout. There is no +known work-around for this (and it usually should not case any troubles.

    Sample:

    - [manual index] [rsyslog site]

    This documentation is part of the rsyslog diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 4fbf2ef4..42ed834f 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -1,5 +1,5 @@ /* omhdfs.c - * This is the implementation of the build-in file output module. + * This is an output module to support Hadoop's HDFS. * * NOTE: read comments in module-template.h to understand how this file * works! -- cgit v1.2.3 From 2cc5fbd49fdca20cdeb8eb81455bc1baef627a77 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 09:43:34 +0200 Subject: bugfix/imuxsock: system log socket was deleted in systemd-mode, too related bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=200 Note that this may not be the ultimate fix. I could not definitely identify the root cause of the problem and so I did some changes that I *think* (but could not verify) that resolved the issue. --- plugins/imuxsock/imuxsock.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 24bcebb7..906e28e6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -139,6 +139,7 @@ static int startIndexUxLocalSockets; /* process fd from that index on (used to * read-only after startup */ static int nfd = 1; /* number of Unix sockets open / read-only after startup */ +static int bSysSockFromSystemd = 0; /* Did we receive the system socket from systemd? */ /* config settings */ static int bOmitLocalLogging = 0; @@ -368,6 +369,7 @@ openLogSocket(lstn_t *pLstn) return -1; if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { + bSysSockFromSystemd = 1; /* set default */ int r; /* System log socket code. Check whether an FD was passed in from systemd. If @@ -396,6 +398,7 @@ openLogSocket(lstn_t *pLstn) errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } + bSysSockFromSystemd = 1; /* indicate we got the socket from systemd */ } else { CHKiRet(createLogSocket(pLstn)); } @@ -415,15 +418,6 @@ openLogSocket(lstn_t *pLstn) } } -#if 0 - // SO_TIMESTAMP currently does not work for an unknown reason. Any help is appreciated! - one = 1; - if(setsockopt(pLstn->fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", pLstn->sockName); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } -#endif - finalize_it: if(iRet != RS_RET_OK) { close(pLstn->fd); @@ -802,13 +796,16 @@ CODESTARTafterRun /* Clean-up files. If systemd passed us a socket it is * systemd's job to clean it up.*/ - if (sd_listen_fds(0) > 0) + if(bSysSockFromSystemd) { + DBGPRINTF("imuxsock: got system socket from systemd, not unlinking it\n"); i = 1; - else + } else i = startIndexUxLocalSockets; for(; i < nfd; i++) - if (listeners[i].sockName && listeners[i].fd != -1) + if (listeners[i].sockName && listeners[i].fd != -1) { + DBGPRINTF("imuxsock: unlinking unix socket file[%d] %s\n", i, listeners[i].sockName); unlink((char*) listeners[i].sockName); + } /* free no longer needed string */ free(pLogSockName); free(pLogHostName); -- cgit v1.2.3 From 8e51241a50531dc10d00e54aee00a24fc6760811 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Sep 2010 02:01:10 +0200 Subject: systemd: install service and socket unit files This adds a systemd socket and service unit file to the default install if systemd is found or explicitly enabled in ./configure. Patch is against current git v5-devel. --- .gitignore | 2 ++ Makefile.am | 25 +++++++++++++++++++++++-- configure.ac | 10 ++++++++++ rsyslog.service.in | 8 ++++++++ rsyslog.socket | 8 ++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 rsyslog.service.in create mode 100644 rsyslog.socket diff --git a/.gitignore b/.gitignore index bc362e7f..55f069ee 100644 --- a/.gitignore +++ b/.gitignore @@ -25,9 +25,11 @@ install-sh missing compile rsyslogd +rsyslog.service *.orig rg.conf* *.swp +*.cache # some common names I use during development utils tmp* diff --git a/Makefile.am b/Makefile.am index bc6d8dd0..f3dca447 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,6 +39,25 @@ lmgssutil_la_LDFLAGS = -module -avoid-version lmgssutil_la_LIBADD = $(GSS_LIBS) endif +# +# systemd support +# +if HAVE_SYSTEMD + +dist_systemdsystemunit_DATA = \ + rsyslog.socket + +nodist_systemdsystemunit_DATA = \ + rsyslog.service + +CLEANFILES = \ + rsyslog.service + +%.service: %.service.in + $(AM_V_GEN)sed -e 's,@sbindir\@,$(sbindir),g' $< > $@ + +endif + EXTRA_DIST = \ freebsd/rsyslogd \ slackware/rc.rsyslogd \ @@ -47,7 +66,8 @@ EXTRA_DIST = \ COPYING.LESSER \ contrib/gnutls/ca.pem \ contrib/gnutls/cert.pem \ - contrib/gnutls/key.pem + contrib/gnutls/key.pem \ + rsyslog.service.in SUBDIRS = doc runtime . plugins/immark plugins/imuxsock plugins/imtcp plugins/imudp plugins/omtesting @@ -196,5 +216,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-shave \ --enable-extended-tests \ --enable-impstats \ - --enable-memcheck + --enable-memcheck \ + --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 1050981b..16454d00 100644 --- a/configure.ac +++ b/configure.ac @@ -360,6 +360,16 @@ if test "$enable_unlimited_select" = "yes"; then fi +# support for systemd unit files +AC_ARG_WITH([systemdsystemunitdir], + AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files]), + [], [with_systemdsystemunitdir=$($PKG_CONFIG --variable=systemdsystemunitdir systemd)]) +if test "x$with_systemdsystemunitdir" != xno; then + AC_SUBST([systemdsystemunitdir], [$with_systemdsystemunitdir]) +fi +AM_CONDITIONAL(HAVE_SYSTEMD, [test -n "$with_systemdsystemunitdir" -a "x$with_systemdsystemunitdir" != xno ]) + + # debug AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], diff --git a/rsyslog.service.in b/rsyslog.service.in new file mode 100644 index 00000000..b660f6bc --- /dev/null +++ b/rsyslog.service.in @@ -0,0 +1,8 @@ +[Unit] +Description=System Logging Service + +[Service] +ExecStart=@sbindir@/rsyslogd -n -c 4 + +[Install] +WantedBy=multi-user.target diff --git a/rsyslog.socket b/rsyslog.socket new file mode 100644 index 00000000..0cd86054 --- /dev/null +++ b/rsyslog.socket @@ -0,0 +1,8 @@ +[Unit] +Description=Syslog Socket + +[Socket] +ListenDatagram=/dev/log + +[Install] +WantedBy=sockets.target -- cgit v1.2.3 From b3aa4c696a3e8dd97b549afcf49959173b30fe6a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 12:30:43 +0200 Subject: minor: corrected -c param in systemd startup files --- rsyslog.service.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsyslog.service.in b/rsyslog.service.in index b660f6bc..ea966cc3 100644 --- a/rsyslog.service.in +++ b/rsyslog.service.in @@ -2,7 +2,7 @@ Description=System Logging Service [Service] -ExecStart=@sbindir@/rsyslogd -n -c 4 +ExecStart=@sbindir@/rsyslogd -n -c5 [Install] WantedBy=multi-user.target -- cgit v1.2.3 From e40cb595a2da943483124fae8f215a397add9fca Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 13:58:41 +0200 Subject: omhdfs: support for HUP added --- plugins/omhdfs/omhdfs.c | 42 ++++- runtime/Makefile.am | 6 +- runtime/hashtable.c | 313 ++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.c | 313 ---------------------------------- runtime/hashtable/hashtable_itr.c | 188 -------------------- runtime/hashtable/hashtable_itr.h | 112 ------------ runtime/hashtable/hashtable_private.h | 86 ---------- runtime/hashtable_itr.c | 190 +++++++++++++++++++++ runtime/hashtable_itr.h | 112 ++++++++++++ runtime/hashtable_private.h | 86 ++++++++++ 10 files changed, 738 insertions(+), 710 deletions(-) create mode 100644 runtime/hashtable.c delete mode 100644 runtime/hashtable/hashtable.c delete mode 100644 runtime/hashtable/hashtable_itr.c delete mode 100644 runtime/hashtable/hashtable_itr.h delete mode 100644 runtime/hashtable/hashtable_private.h create mode 100644 runtime/hashtable_itr.c create mode 100644 runtime/hashtable_itr.h create mode 100644 runtime/hashtable_private.h diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 42ed834f..eefea722 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -47,6 +47,7 @@ #include "unicode-helper.h" #include "errmsg.h" #include "hashtable.h" +#include "hashtable_itr.h" MODULE_TYPE_OUTPUT @@ -200,7 +201,6 @@ fileObjDestruct(file_t **ppFile) static void fileObjDestruct4Hashtable(void *ptr) { - dbgprintf("omfile: fileObjDestruct4Hashtable called\n"); file_t *pFile = (file_t*) ptr; fileObjDestruct(&pFile); } @@ -214,6 +214,7 @@ fileOpen(file_t *pFile) assert(pFile->fh == NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); + DBGPRINTF("omhdfs: try to connect to HDFS at host '%s', port %d\n", pFile->hdfsHost, pFile->hdfsPort); pFile->fs = hdfsConnect(pFile->hdfsHost, pFile->hdfsPort); @@ -256,6 +257,17 @@ fileWrite(file_t *pFile, uchar *buf) assert(pFile->fh != NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); + + /* open file if not open. This must be done *here* and while mutex-protected + * because of HUP handling (which is async to normal processing!). + */ + if(pFile->fh == NULL) { + fileOpen(pFile); + if(pFile->fh == NULL) { + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } + lenWrite = strlen((char*) buf); tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { @@ -321,15 +333,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->pFile->name); - if(pData->pFile->fh == NULL) { - fileOpen(pData->pFile); - if(pData->pFile->fh == NULL) { - ABORT_FINALIZE(RS_RET_SUSPENDED); - } - } + DBGPRINTF("omuxsock: action to to write to %s\n", pData->pFile->name); iRet = fileWrite(pData->pFile, ppString[0]); -finalize_it: ENDdoAction @@ -382,6 +387,24 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct +BEGINdoHUP + file_t *pFile; + struct hashtable_itr *itr; +CODESTARTdoHUP + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(files); + if(hashtable_count(files) > 0) + { + do { + pFile = (file_t *) hashtable_iterator_value(itr); + fileClose(pFile); + DBGPRINTF("imuxsock: HUP, closing file %s\n", pFile->name); + } while (hashtable_iterator_advance(itr)); + } +ENDdoHUP + + /* Reset config variables for this module to default values. * rgerhards, 2007-07-17 */ @@ -415,6 +438,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_doHUP ENDqueryEtryPt diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 5a0c4437..93817e75 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -95,9 +95,11 @@ librsyslog_la_SOURCES = \ ../parse.c \ ../parse.h \ \ - hashtable/hashtable.c \ + hashtable.c \ hashtable.h \ - hashtable/hashtable_private.h \ + hashtable_itr.c \ + hashtable_itr.h \ + hashtable_private.h \ \ ../outchannel.c \ ../outchannel.h \ diff --git a/runtime/hashtable.c b/runtime/hashtable.c new file mode 100644 index 00000000..41fc60fe --- /dev/null +++ b/runtime/hashtable.c @@ -0,0 +1,313 @@ +/* Copyright (C) 2004 Christopher Clark */ +/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include +#include +#include +#include + +/* +Credit for primes table: Aaron Krowne + http://br.endernet.org/~akrowne/ + http://planetmath.org/encyclopedia/GoodHashTablePrimes.html +*/ +static const unsigned int primes[] = { +53, 97, 193, 389, +769, 1543, 3079, 6151, +12289, 24593, 49157, 98317, +196613, 393241, 786433, 1572869, +3145739, 6291469, 12582917, 25165843, +50331653, 100663319, 201326611, 402653189, +805306457, 1610612741 +}; +const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); +const float max_load_factor = 0.65; + +/*****************************************************************************/ +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashf) (void*), + int (*eqf) (void*,void*), void (*dest)(void*)) +{ + struct hashtable *h; + unsigned int pindex, size = primes[0]; + /* Check requested hashtable isn't too large */ + if (minsize > (1u << 30)) return NULL; + /* Enforce size as prime */ + for (pindex=0; pindex < prime_table_length; pindex++) { + if (primes[pindex] > minsize) { size = primes[pindex]; break; } + } + h = (struct hashtable *)malloc(sizeof(struct hashtable)); + if (NULL == h) return NULL; /*oom*/ + h->table = (struct entry **)malloc(sizeof(struct entry*) * size); + if (NULL == h->table) { free(h); return NULL; } /*oom*/ + memset(h->table, 0, size * sizeof(struct entry *)); + h->tablelength = size; + h->primeindex = pindex; + h->entrycount = 0; + h->hashfn = hashf; + h->eqfn = eqf; + h->dest = dest; + h->loadlimit = (unsigned int) ceil(size * max_load_factor); + return h; +} + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k) +{ + /* Aim to protect against poor hash functions by adding logic here + * - logic taken from java 1.4 hashtable source */ + unsigned int i = h->hashfn(k); + i += ~(i << 9); + i ^= ((i >> 14) | (i << 18)); /* >>> */ + i += (i << 4); + i ^= ((i >> 10) | (i << 22)); /* >>> */ + return i; +} + +/*****************************************************************************/ +static int +hashtable_expand(struct hashtable *h) +{ + /* Double the size of the table to accomodate more entries */ + struct entry **newtable; + struct entry *e; + struct entry **pE; + unsigned int newsize, i, idx; + /* Check we're not hitting max capacity */ + if (h->primeindex == (prime_table_length - 1)) return 0; + newsize = primes[++(h->primeindex)]; + + newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); + if (NULL != newtable) + { + memset(newtable, 0, newsize * sizeof(struct entry *)); + /* This algorithm is not 'stable'. ie. it reverses the list + * when it transfers entries between the tables */ + for (i = 0; i < h->tablelength; i++) { + while (NULL != (e = h->table[i])) { + h->table[i] = e->next; + idx = indexFor(newsize,e->h); + e->next = newtable[idx]; + newtable[idx] = e; + } + } + free(h->table); + h->table = newtable; + } + /* Plan B: realloc instead */ + else + { + newtable = (struct entry **) + realloc(h->table, newsize * sizeof(struct entry *)); + if (NULL == newtable) { (h->primeindex)--; return 0; } + h->table = newtable; + memset(newtable[h->tablelength], 0, newsize - h->tablelength); + for (i = 0; i < h->tablelength; i++) { + for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { + idx = indexFor(newsize,e->h); + if (idx == i) + { + pE = &(e->next); + } + else + { + *pE = e->next; + e->next = newtable[idx]; + newtable[idx] = e; + } + } + } + } + h->tablelength = newsize; + h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + return -1; +} + +/*****************************************************************************/ +unsigned int +hashtable_count(struct hashtable *h) +{ + return h->entrycount; +} + +/*****************************************************************************/ +int +hashtable_insert(struct hashtable *h, void *k, void *v) +{ + /* This method allows duplicate keys - but they shouldn't be used */ + unsigned int idx; + struct entry *e; + if (++(h->entrycount) > h->loadlimit) + { + /* Ignore the return value. If expand fails, we should + * still try cramming just this value into the existing table + * -- we may not have memory for a larger table, but one more + * element may be ok. Next time we insert, we'll try expanding again.*/ + hashtable_expand(h); + } + e = (struct entry *)malloc(sizeof(struct entry)); + if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ + e->h = hash(h,k); + idx = indexFor(h->tablelength,e->h); + e->k = k; + e->v = v; + e->next = h->table[idx]; + h->table[idx] = e; + return -1; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_search(struct hashtable *h, void *k) +{ + struct entry *e; + unsigned int hashvalue, idx; + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hashvalue); + e = h->table[idx]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_remove(struct hashtable *h, void *k) +{ + /* TODO: consider compacting the table when the load factor drops enough, + * or provide a 'compact' method. */ + + struct entry *e; + struct entry **pE; + void *v; + unsigned int hashvalue, idx; + + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hash(h,k)); + pE = &(h->table[idx]); + e = *pE; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + *pE = e->next; + h->entrycount--; + v = e->v; + freekey(e->k); + free(e); + return v; + } + pE = &(e->next); + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +/* destroy */ +void +hashtable_destroy(struct hashtable *h, int free_values) +{ + unsigned int i; + struct entry *e, *f; + struct entry **table = h->table; + if (free_values) + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { + f = e; + e = e->next; + freekey(f->k); + if(h->dest == NULL) + free(f->v); + else + h->dest(f->v); + free(f); + } + } + } + else + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f); } + } + } + free(h->table); + free(h); +} + +/* some generic hash functions */ + +/* one provided by Aaaron Wiebe based on perl's hashng algorithm + * (so probably pretty generic). Not for excessively large strings! + */ +unsigned int +hash_from_string(void *k) +{ + int len; + char *rkey = (char*) k; + unsigned hashval = 1; + + len = (int) strlen(rkey); + while (len--) + hashval = hashval * 33 + *rkey++; + + return hashval; +} + + +int +key_equals_string(void *key1, void *key2) +{ + /* we must return true IF the keys are equal! */ + return !strcmp(key1, key2); +} + + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c deleted file mode 100644 index 41fc60fe..00000000 --- a/runtime/hashtable/hashtable.c +++ /dev/null @@ -1,313 +0,0 @@ -/* Copyright (C) 2004 Christopher Clark */ -/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ - -#include "hashtable.h" -#include "hashtable_private.h" -#include -#include -#include -#include - -/* -Credit for primes table: Aaron Krowne - http://br.endernet.org/~akrowne/ - http://planetmath.org/encyclopedia/GoodHashTablePrimes.html -*/ -static const unsigned int primes[] = { -53, 97, 193, 389, -769, 1543, 3079, 6151, -12289, 24593, 49157, 98317, -196613, 393241, 786433, 1572869, -3145739, 6291469, 12582917, 25165843, -50331653, 100663319, 201326611, 402653189, -805306457, 1610612741 -}; -const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); -const float max_load_factor = 0.65; - -/*****************************************************************************/ -struct hashtable * -create_hashtable(unsigned int minsize, - unsigned int (*hashf) (void*), - int (*eqf) (void*,void*), void (*dest)(void*)) -{ - struct hashtable *h; - unsigned int pindex, size = primes[0]; - /* Check requested hashtable isn't too large */ - if (minsize > (1u << 30)) return NULL; - /* Enforce size as prime */ - for (pindex=0; pindex < prime_table_length; pindex++) { - if (primes[pindex] > minsize) { size = primes[pindex]; break; } - } - h = (struct hashtable *)malloc(sizeof(struct hashtable)); - if (NULL == h) return NULL; /*oom*/ - h->table = (struct entry **)malloc(sizeof(struct entry*) * size); - if (NULL == h->table) { free(h); return NULL; } /*oom*/ - memset(h->table, 0, size * sizeof(struct entry *)); - h->tablelength = size; - h->primeindex = pindex; - h->entrycount = 0; - h->hashfn = hashf; - h->eqfn = eqf; - h->dest = dest; - h->loadlimit = (unsigned int) ceil(size * max_load_factor); - return h; -} - -/*****************************************************************************/ -unsigned int -hash(struct hashtable *h, void *k) -{ - /* Aim to protect against poor hash functions by adding logic here - * - logic taken from java 1.4 hashtable source */ - unsigned int i = h->hashfn(k); - i += ~(i << 9); - i ^= ((i >> 14) | (i << 18)); /* >>> */ - i += (i << 4); - i ^= ((i >> 10) | (i << 22)); /* >>> */ - return i; -} - -/*****************************************************************************/ -static int -hashtable_expand(struct hashtable *h) -{ - /* Double the size of the table to accomodate more entries */ - struct entry **newtable; - struct entry *e; - struct entry **pE; - unsigned int newsize, i, idx; - /* Check we're not hitting max capacity */ - if (h->primeindex == (prime_table_length - 1)) return 0; - newsize = primes[++(h->primeindex)]; - - newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); - if (NULL != newtable) - { - memset(newtable, 0, newsize * sizeof(struct entry *)); - /* This algorithm is not 'stable'. ie. it reverses the list - * when it transfers entries between the tables */ - for (i = 0; i < h->tablelength; i++) { - while (NULL != (e = h->table[i])) { - h->table[i] = e->next; - idx = indexFor(newsize,e->h); - e->next = newtable[idx]; - newtable[idx] = e; - } - } - free(h->table); - h->table = newtable; - } - /* Plan B: realloc instead */ - else - { - newtable = (struct entry **) - realloc(h->table, newsize * sizeof(struct entry *)); - if (NULL == newtable) { (h->primeindex)--; return 0; } - h->table = newtable; - memset(newtable[h->tablelength], 0, newsize - h->tablelength); - for (i = 0; i < h->tablelength; i++) { - for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { - idx = indexFor(newsize,e->h); - if (idx == i) - { - pE = &(e->next); - } - else - { - *pE = e->next; - e->next = newtable[idx]; - newtable[idx] = e; - } - } - } - } - h->tablelength = newsize; - h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); - return -1; -} - -/*****************************************************************************/ -unsigned int -hashtable_count(struct hashtable *h) -{ - return h->entrycount; -} - -/*****************************************************************************/ -int -hashtable_insert(struct hashtable *h, void *k, void *v) -{ - /* This method allows duplicate keys - but they shouldn't be used */ - unsigned int idx; - struct entry *e; - if (++(h->entrycount) > h->loadlimit) - { - /* Ignore the return value. If expand fails, we should - * still try cramming just this value into the existing table - * -- we may not have memory for a larger table, but one more - * element may be ok. Next time we insert, we'll try expanding again.*/ - hashtable_expand(h); - } - e = (struct entry *)malloc(sizeof(struct entry)); - if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ - e->h = hash(h,k); - idx = indexFor(h->tablelength,e->h); - e->k = k; - e->v = v; - e->next = h->table[idx]; - h->table[idx] = e; - return -1; -} - -/*****************************************************************************/ -void * /* returns value associated with key */ -hashtable_search(struct hashtable *h, void *k) -{ - struct entry *e; - unsigned int hashvalue, idx; - hashvalue = hash(h,k); - idx = indexFor(h->tablelength,hashvalue); - e = h->table[idx]; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; - e = e->next; - } - return NULL; -} - -/*****************************************************************************/ -void * /* returns value associated with key */ -hashtable_remove(struct hashtable *h, void *k) -{ - /* TODO: consider compacting the table when the load factor drops enough, - * or provide a 'compact' method. */ - - struct entry *e; - struct entry **pE; - void *v; - unsigned int hashvalue, idx; - - hashvalue = hash(h,k); - idx = indexFor(h->tablelength,hash(h,k)); - pE = &(h->table[idx]); - e = *pE; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) - { - *pE = e->next; - h->entrycount--; - v = e->v; - freekey(e->k); - free(e); - return v; - } - pE = &(e->next); - e = e->next; - } - return NULL; -} - -/*****************************************************************************/ -/* destroy */ -void -hashtable_destroy(struct hashtable *h, int free_values) -{ - unsigned int i; - struct entry *e, *f; - struct entry **table = h->table; - if (free_values) - { - for (i = 0; i < h->tablelength; i++) - { - e = table[i]; - while (NULL != e) - { - f = e; - e = e->next; - freekey(f->k); - if(h->dest == NULL) - free(f->v); - else - h->dest(f->v); - free(f); - } - } - } - else - { - for (i = 0; i < h->tablelength; i++) - { - e = table[i]; - while (NULL != e) - { f = e; e = e->next; freekey(f->k); free(f); } - } - } - free(h->table); - free(h); -} - -/* some generic hash functions */ - -/* one provided by Aaaron Wiebe based on perl's hashng algorithm - * (so probably pretty generic). Not for excessively large strings! - */ -unsigned int -hash_from_string(void *k) -{ - int len; - char *rkey = (char*) k; - unsigned hashval = 1; - - len = (int) strlen(rkey); - while (len--) - hashval = hashval * 33 + *rkey++; - - return hashval; -} - - -int -key_equals_string(void *key1, void *key2) -{ - /* we must return true IF the keys are equal! */ - return !strcmp(key1, key2); -} - - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_itr.c b/runtime/hashtable/hashtable_itr.c deleted file mode 100644 index 5dced841..00000000 --- a/runtime/hashtable/hashtable_itr.c +++ /dev/null @@ -1,188 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#include "hashtable.h" -#include "hashtable_private.h" -#include "hashtable_itr.h" -#include /* defines NULL */ - -/*****************************************************************************/ -/* hashtable_iterator - iterator constructor */ - -struct hashtable_itr * -hashtable_iterator(struct hashtable *h) -{ - unsigned int i, tablelength; - struct hashtable_itr *itr = (struct hashtable_itr *) - malloc(sizeof(struct hashtable_itr)); - if (NULL == itr) return NULL; - itr->h = h; - itr->e = NULL; - itr->parent = NULL; - tablelength = h->tablelength; - itr->index = tablelength; - if (0 == h->entrycount) return itr; - - for (i = 0; i < tablelength; i++) - { - if (NULL != h->table[i]) - { - itr->e = h->table[i]; - itr->index = i; - break; - } - } - return itr; -} - -/*****************************************************************************/ -/* key - return the key of the (key,value) pair at the current position */ -/* value - return the value of the (key,value) pair at the current position */ - -void * -hashtable_iterator_key(struct hashtable_itr *i) -{ return i->e->k; } - -void * -hashtable_iterator_value(struct hashtable_itr *i) -{ return i->e->v; } - -/*****************************************************************************/ -/* advance - advance the iterator to the next element - * returns zero if advanced to end of table */ - -int -hashtable_iterator_advance(struct hashtable_itr *itr) -{ - unsigned int j,tablelength; - struct entry **table; - struct entry *next; - if (NULL == itr->e) return 0; /* stupidity check */ - - next = itr->e->next; - if (NULL != next) - { - itr->parent = itr->e; - itr->e = next; - return -1; - } - tablelength = itr->h->tablelength; - itr->parent = NULL; - if (tablelength <= (j = ++(itr->index))) - { - itr->e = NULL; - return 0; - } - table = itr->h->table; - while (NULL == (next = table[j])) - { - if (++j >= tablelength) - { - itr->index = tablelength; - itr->e = NULL; - return 0; - } - } - itr->index = j; - itr->e = next; - return -1; -} - -/*****************************************************************************/ -/* remove - remove the entry at the current iterator position - * and advance the iterator, if there is a successive - * element. - * If you want the value, read it before you remove: - * beware memory leaks if you don't. - * Returns zero if end of iteration. */ - -int -hashtable_iterator_remove(struct hashtable_itr *itr) -{ - struct entry *remember_e, *remember_parent; - int ret; - - /* Do the removal */ - if (NULL == (itr->parent)) - { - /* element is head of a chain */ - itr->h->table[itr->index] = itr->e->next; - } else { - /* element is mid-chain */ - itr->parent->next = itr->e->next; - } - /* itr->e is now outside the hashtable */ - remember_e = itr->e; - itr->h->entrycount--; - freekey(remember_e->k); - - /* Advance the iterator, correcting the parent */ - remember_parent = itr->parent; - ret = hashtable_iterator_advance(itr); - if (itr->parent == remember_e) { itr->parent = remember_parent; } - free(remember_e); - return ret; -} - -/*****************************************************************************/ -int /* returns zero if not found */ -hashtable_iterator_search(struct hashtable_itr *itr, - struct hashtable *h, void *k) -{ - struct entry *e, *parent; - unsigned int hashvalue, index; - - hashvalue = hash(h,k); - index = indexFor(h->tablelength,hashvalue); - - e = h->table[index]; - parent = NULL; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) - { - itr->index = index; - itr->e = e; - itr->parent = parent; - itr->h = h; - return -1; - } - parent = e; - e = e->next; - } - return 0; -} - - -/* - * Copyright (c) 2002, 2004, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_itr.h b/runtime/hashtable/hashtable_itr.h deleted file mode 100644 index eea699a7..00000000 --- a/runtime/hashtable/hashtable_itr.h +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#ifndef __HASHTABLE_ITR_CWC22__ -#define __HASHTABLE_ITR_CWC22__ -#include "hashtable.h" -#include "hashtable_private.h" /* needed to enable inlining */ - -/*****************************************************************************/ -/* This struct is only concrete here to allow the inlining of two of the - * accessor functions. */ -struct hashtable_itr -{ - struct hashtable *h; - struct entry *e; - struct entry *parent; - unsigned int index; -}; - - -/*****************************************************************************/ -/* hashtable_iterator - */ - -struct hashtable_itr * -hashtable_iterator(struct hashtable *h); - -/*****************************************************************************/ -/* hashtable_iterator_key - * - return the value of the (key,value) pair at the current position */ - -extern inline void * -hashtable_iterator_key(struct hashtable_itr *i) -{ - return i->e->k; -} - -/*****************************************************************************/ -/* value - return the value of the (key,value) pair at the current position */ - -extern inline void * -hashtable_iterator_value(struct hashtable_itr *i) -{ - return i->e->v; -} - -/*****************************************************************************/ -/* advance - advance the iterator to the next element - * returns zero if advanced to end of table */ - -int -hashtable_iterator_advance(struct hashtable_itr *itr); - -/*****************************************************************************/ -/* remove - remove current element and advance the iterator to the next element - * NB: if you need the value to free it, read it before - * removing. ie: beware memory leaks! - * returns zero if advanced to end of table */ - -int -hashtable_iterator_remove(struct hashtable_itr *itr); - -/*****************************************************************************/ -/* search - overwrite the supplied iterator, to point to the entry - * matching the supplied key. - h points to the hashtable to be searched. - * returns zero if not found. */ -int -hashtable_iterator_search(struct hashtable_itr *itr, - struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ -int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ -{ \ - return (hashtable_iterator_search(i,h,k)); \ -} - - - -#endif /* __HASHTABLE_ITR_CWC22__*/ - -/* - * Copyright (c) 2002, 2004, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h deleted file mode 100644 index 10b82da4..00000000 --- a/runtime/hashtable/hashtable_private.h +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#ifndef __HASHTABLE_PRIVATE_CWC22_H__ -#define __HASHTABLE_PRIVATE_CWC22_H__ - -#include "hashtable.h" - -/*****************************************************************************/ -struct entry -{ - void *k, *v; - unsigned int h; - struct entry *next; -}; - -struct hashtable { - unsigned int tablelength; - struct entry **table; - unsigned int entrycount; - unsigned int loadlimit; - unsigned int primeindex; - unsigned int (*hashfn) (void *k); - int (*eqfn) (void *k1, void *k2); - void (*dest) (void *v); /* destructor for values, if NULL use free() */ -}; - -/*****************************************************************************/ -unsigned int -hash(struct hashtable *h, void *k); - -/*****************************************************************************/ -/* indexFor */ -static inline unsigned int -indexFor(unsigned int tablelength, unsigned int hashvalue) { - return (hashvalue % tablelength); -}; - -/* Only works if tablelength == 2^N */ -/*static inline unsigned int -indexFor(unsigned int tablelength, unsigned int hashvalue) -{ - return (hashvalue & (tablelength - 1u)); -} -*/ - -/*****************************************************************************/ -#define freekey(X) free(X) -/*define freekey(X) ; */ - - -/*****************************************************************************/ - -#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable_itr.c b/runtime/hashtable_itr.c new file mode 100644 index 00000000..967287f1 --- /dev/null +++ b/runtime/hashtable_itr.c @@ -0,0 +1,190 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_itr.h" +#include /* defines NULL */ + +/*****************************************************************************/ +/* hashtable_iterator - iterator constructor */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h) +{ + unsigned int i, tablelength; + struct hashtable_itr *itr = (struct hashtable_itr *) + malloc(sizeof(struct hashtable_itr)); + if (NULL == itr) return NULL; + itr->h = h; + itr->e = NULL; + itr->parent = NULL; + tablelength = h->tablelength; + itr->index = tablelength; + if (0 == h->entrycount) return itr; + + for (i = 0; i < tablelength; i++) + { + if (NULL != h->table[i]) + { + itr->e = h->table[i]; + itr->index = i; + break; + } + } + return itr; +} + +/*****************************************************************************/ +/* key - return the key of the (key,value) pair at the current position */ +/* value - return the value of the (key,value) pair at the current position */ + +#if 0 /* these are now inline functions! */ +void * +hashtable_iterator_key(struct hashtable_itr *i) +{ return i->e->k; } + +void * +hashtable_iterator_value(struct hashtable_itr *i) +{ return i->e->v; } +#endif + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr) +{ + unsigned int j,tablelength; + struct entry **table; + struct entry *next; + if (NULL == itr->e) return 0; /* stupidity check */ + + next = itr->e->next; + if (NULL != next) + { + itr->parent = itr->e; + itr->e = next; + return -1; + } + tablelength = itr->h->tablelength; + itr->parent = NULL; + if (tablelength <= (j = ++(itr->index))) + { + itr->e = NULL; + return 0; + } + table = itr->h->table; + while (NULL == (next = table[j])) + { + if (++j >= tablelength) + { + itr->index = tablelength; + itr->e = NULL; + return 0; + } + } + itr->index = j; + itr->e = next; + return -1; +} + +/*****************************************************************************/ +/* remove - remove the entry at the current iterator position + * and advance the iterator, if there is a successive + * element. + * If you want the value, read it before you remove: + * beware memory leaks if you don't. + * Returns zero if end of iteration. */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr) +{ + struct entry *remember_e, *remember_parent; + int ret; + + /* Do the removal */ + if (NULL == (itr->parent)) + { + /* element is head of a chain */ + itr->h->table[itr->index] = itr->e->next; + } else { + /* element is mid-chain */ + itr->parent->next = itr->e->next; + } + /* itr->e is now outside the hashtable */ + remember_e = itr->e; + itr->h->entrycount--; + freekey(remember_e->k); + + /* Advance the iterator, correcting the parent */ + remember_parent = itr->parent; + ret = hashtable_iterator_advance(itr); + if (itr->parent == remember_e) { itr->parent = remember_parent; } + free(remember_e); + return ret; +} + +/*****************************************************************************/ +int /* returns zero if not found */ +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k) +{ + struct entry *e, *parent; + unsigned int hashvalue, index; + + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + + e = h->table[index]; + parent = NULL; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + itr->index = index; + itr->e = e; + itr->parent = parent; + itr->h = h; + return -1; + } + parent = e; + e = e->next; + } + return 0; +} + + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable_itr.h b/runtime/hashtable_itr.h new file mode 100644 index 00000000..1c206b6e --- /dev/null +++ b/runtime/hashtable_itr.h @@ -0,0 +1,112 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_ITR_CWC22__ +#define __HASHTABLE_ITR_CWC22__ +#include "hashtable.h" +#include "hashtable_private.h" /* needed to enable inlining */ + +/*****************************************************************************/ +/* This struct is only concrete here to allow the inlining of two of the + * accessor functions. */ +struct hashtable_itr +{ + struct hashtable *h; + struct entry *e; + struct entry *parent; + unsigned int index; +}; + + +/*****************************************************************************/ +/* hashtable_iterator + */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h); + +/*****************************************************************************/ +/* hashtable_iterator_key + * - return the value of the (key,value) pair at the current position */ + +static inline void * +hashtable_iterator_key(struct hashtable_itr *i) +{ + return i->e->k; +} + +/*****************************************************************************/ +/* value - return the value of the (key,value) pair at the current position */ + +static inline void * +hashtable_iterator_value(struct hashtable_itr *i) +{ + return i->e->v; +} + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* remove - remove current element and advance the iterator to the next element + * NB: if you need the value to free it, read it before + * removing. ie: beware memory leaks! + * returns zero if advanced to end of table */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* search - overwrite the supplied iterator, to point to the entry + * matching the supplied key. + h points to the hashtable to be searched. + * returns zero if not found. */ +int +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ +int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ +{ \ + return (hashtable_iterator_search(i,h,k)); \ +} + + + +#endif /* __HASHTABLE_ITR_CWC22__*/ + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable_private.h b/runtime/hashtable_private.h new file mode 100644 index 00000000..10b82da4 --- /dev/null +++ b/runtime/hashtable_private.h @@ -0,0 +1,86 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_PRIVATE_CWC22_H__ +#define __HASHTABLE_PRIVATE_CWC22_H__ + +#include "hashtable.h" + +/*****************************************************************************/ +struct entry +{ + void *k, *v; + unsigned int h; + struct entry *next; +}; + +struct hashtable { + unsigned int tablelength; + struct entry **table; + unsigned int entrycount; + unsigned int loadlimit; + unsigned int primeindex; + unsigned int (*hashfn) (void *k); + int (*eqfn) (void *k1, void *k2); + void (*dest) (void *v); /* destructor for values, if NULL use free() */ +}; + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k); + +/*****************************************************************************/ +/* indexFor */ +static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) { + return (hashvalue % tablelength); +}; + +/* Only works if tablelength == 2^N */ +/*static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) +{ + return (hashvalue & (tablelength - 1u)); +} +*/ + +/*****************************************************************************/ +#define freekey(X) free(X) +/*define freekey(X) ; */ + + +/*****************************************************************************/ + +#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ -- cgit v1.2.3 From 4b9a92bc725f7436b7958e673a9665a90b548e86 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 12:46:39 +0000 Subject: omhdfs: fixed some issues with previous commit --- plugins/omhdfs/omhdfs.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index eefea722..b075432d 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -254,7 +254,6 @@ fileWrite(file_t *pFile, uchar *buf) size_t lenWrite; DEFiRet; - assert(pFile->fh != NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); @@ -391,17 +390,18 @@ BEGINdoHUP file_t *pFile; struct hashtable_itr *itr; CODESTARTdoHUP - /* Iterator constructor only returns a valid iterator if - * the hashtable is not empty */ - itr = hashtable_iterator(files); - if(hashtable_count(files) > 0) - { - do { - pFile = (file_t *) hashtable_iterator_value(itr); - fileClose(pFile); - DBGPRINTF("imuxsock: HUP, closing file %s\n", pFile->name); - } while (hashtable_iterator_advance(itr)); - } + DBGPRINTF("omhdfs: HUP received (file count %d)\n", hashtable_count(files)); + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(files); + if(hashtable_count(files) > 0) + { + do { + pFile = (file_t *) hashtable_iterator_value(itr); + fileClose(pFile); + DBGPRINTF("omhdfs: HUP, closing file %s\n", pFile->name); + } while (hashtable_iterator_advance(itr)); + } ENDdoHUP -- cgit v1.2.3 From bbf38b082a17c77dfa86dcdbd0b7b7b6d36c07b6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 16:17:49 +0200 Subject: imuxsock: fixed dumb mistake of last commit I left a testing setting in, resulting in /dev/log never being deleted... --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 906e28e6..75a6b00b 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -369,7 +369,7 @@ openLogSocket(lstn_t *pLstn) return -1; if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { - bSysSockFromSystemd = 1; /* set default */ + bSysSockFromSystemd = 0; /* set default */ int r; /* System log socket code. Check whether an FD was passed in from systemd. If -- cgit v1.2.3 From a627ac99ba2c3404ca926a19fb06cbd6f43b53c8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 16:25:49 +0200 Subject: omhdfs: added ability to create non-existing directories in name path --- plugins/omhdfs/omhdfs.c | 128 +++++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index b075432d..9705b7fd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -97,54 +97,41 @@ CODESTARTdbgPrintInstInfo ENDdbgPrintInstInfo - -#if 0 -static void prepareFile(instanceData *pData, uchar *newFileName) +/* note that hdfsFileExists() does not work, so we did our + * own function to see if a pathname exists. Returns 0 if the + * file does not exists, something else otherwise. Note that + * we can also check a directroy (if that matters...) + */ +static int +HDFSFileExists(hdfsFS fs, uchar *name) { - if(access((char*)newFileName, F_OK) == 0) { - /* file already exists */ - pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, - pData->fCreateMode); + int r; + hdfsFileInfo *info; + + info = hdfsGetPathInfo(fs, (char*)name); + /* if things go wrong, we assume it is because the file + * does not exist. We do not get too much information... + */ + if(info == NULL) { + r = 0; } else { - pData->fh = -1; - /* file does not exist, create it (and eventually parent directories */ - if(pData->bCreateDirs) { - /* we fist need to create parent dirs if they are missing - * We do not report any errors here ourselfs but let the code - * fall through to error handler below. - */ - if(makeFileParentDirs(newFileName, strlen((char*)newFileName), - pData->fDirCreateMode, pData->dirUID, - pData->dirGID, pData->bFailOnChown) != 0) { - return; /* we give up */ - } - } - /* no matter if we needed to create directories or not, we now try to create - * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) - */ - pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, - pData->fCreateMode); - if(pData->fh != -1) { - /* check and set uid/gid */ - if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { - /* we need to set owner/group */ - if(fchown(pData->fh, pData->fileUID, - pData->fileGID) != 0) { - if(pData->bFailOnChown) { - int eSave = errno; - close(pData->fh); - pData->fh = -1; - errno = eSave; - } - /* we will silently ignore the chown() failure - * if configured to do so. - */ - } - } - } + r = 1; + hdfsFreeFileInfo(info, 1); } + return r; +} + +static inline rsRetVal +HDFSmkdir(hdfsFS fs, uchar *name) +{ + DEFiRet; + if(hdfsCreateDirectory(fs, (char*)name) == -1) + ABORT_FINALIZE(RS_RET_ERR); + +finalize_it: + RETiRet; } -#endif + /* ---BEGIN FILE OBJECT---------------------------------------------------- */ /* This code handles the "file object". This is split from the actual @@ -195,6 +182,42 @@ fileObjDestruct(file_t **ppFile) return RS_RET_OK; } + +/* check, and potentially create, all names inside a path */ +static rsRetVal +filePrepare(file_t *pFile) +{ + uchar *p; + uchar *pszWork; + size_t len; + DEFiRet; + + if(HDFSFileExists(pFile->fs, pFile->name)) + FINALIZE; + + /* file does not exist, create it (and eventually parent directories */ + if(1) { // check if bCreateDirs + len = ustrlen(pFile->name) + 1; + CHKmalloc(pszWork = MALLOC(sizeof(uchar) * len)); + memcpy(pszWork, pFile->name, len); + for(p = pszWork+1 ; *p ; p++) + if(*p == '/') { + /* temporarily terminate string, create dir and go on */ + *p = '\0'; + if(!HDFSFileExists(pFile->fs, pszWork)) { + CHKiRet(HDFSmkdir(pFile->fs, pszWork)); + } + *p = '/'; + } + free(pszWork); + return 0; + } + +finalize_it: + RETiRet; +} + + /* this function is to be used as destructor for the * hash table code. */ @@ -222,6 +245,9 @@ fileOpen(file_t *pFile) DBGPRINTF("omhdfs: error can not connect to hdfs\n"); ABORT_FINALIZE(RS_RET_SUSPENDED); } + + CHKiRet(filePrepare(pFile)); + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_APPEND, 0, 0, 0); if(pFile->fh == NULL) { /* maybe the file does not exist, so we try to create it now. @@ -271,7 +297,8 @@ fileWrite(file_t *pFile, uchar *buf) tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, "omhdfs: failed to write %s, expected %lu bytes, " - "written %lu\n", pFile->name, lenWrite, (unsigned long) num_written_bytes); + "written %lu\n", pFile->name, (unsigned long) lenWrite, + (unsigned long) num_written_bytes); ABORT_FINALIZE(RS_RET_SUSPENDED); } @@ -410,17 +437,6 @@ ENDdoHUP */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { -/* - fileUID = -1; - fileGID = -1; - dirUID = -1; - dirGID = -1; - bFailOnChown = 1; - iDynaFileCacheSize = 10; - fCreateMode = 0644; - fDirCreateMode = 0700; - bCreateDirs = 1; -*/ hdfsHost = NULL; hdfsPort = 0; return RS_RET_OK; -- cgit v1.2.3 From 6ea4a38079cb1508c103930e327fa92ada97176f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 5 Oct 2010 14:06:27 +0200 Subject: updated ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index f9b79392..055f40f4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,8 @@ Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now supports up to 50 different sockets for input - some code cleanup in imuxsock (consider this a release a major modification, especially if problems show up) +- bugfix: /dev/log was unlinked even when passed in from systemd + in which case it should be preserved as systemd owns it --------------------------------------------------------------------------- Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 - added module impstat to emit periodic statistics on rsyslog counters -- cgit v1.2.3 From b5afa1a5be2909ed13f77ef0e0804bab75325198 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 5 Oct 2010 15:28:46 +0200 Subject: preparing for 5.7.1 --- ChangeLog | 4 +--- doc/manual.html | 2 +- doc/omhdfs.html | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3bb3d882..f552781a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,6 @@ --------------------------------------------------------------------------- -Version 5.7.2 [V5-DEVEL] (rgerhards), 2010-09-?? +Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-10-05 - support for Hadoop's HDFS added (via omhdfs) ---------------------------------------------------------------------------- -Version 5.7.1 [V5-DEVEL] (rgerhards), 2010-09-?? - imuxsock now optionally use SCM_CREDENTIALS to pull the pid from the log socket itself (thanks to Lennart Poettering for the suggesting this feature) diff --git a/doc/manual.html b/doc/manual.html index 08334c5d..6affe746 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 5.7.0 (beta branch) of rsyslog. +

    This documentation is for version 5.7.1 (beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might diff --git a/doc/omhdfs.html b/doc/omhdfs.html index 3849f167..ef7e60c5 100644 --- a/doc/omhdfs.html +++ b/doc/omhdfs.html @@ -5,7 +5,7 @@

    Unix sockets Output Module (omhdfs)

    Module Name:    omhdfs

    -

    Available since:    5.7.2

    +

    Available since:    5.7.1

    Author: Rainer Gerhards <rgerhards@adiscon.com>

    Description:

    This module supports writing message into files on Hadoop's HDFS -- cgit v1.2.3