From 85a0ed1ccefc9bf0d054dac0d5305b1f6cc0fe22 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 6 Feb 2010 12:03:24 +0100 Subject: bugfix: imuxsock incorrectly stated inputname "imudp" Thanks to Ryan Lynch for reporting this. --- ChangeLog | 2 ++ plugins/imuxsock/imuxsock.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 37b70c8f..bfcb1fa6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.5.8 [v4-beta] (rgerhards), 2010-01-?? +- bugfix: imuxsock incorrectly stated inputname "imudp" + Thanks to Ryan Lynch for reporting this. - (slightly) enhanced support for FreeBSD by setting _PATH_MODDIR to the correct value on FreeBSD. Thanks to Cristiano for the patch. diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 424d0904..5567a405 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -70,7 +70,7 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) -static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ +static prop_t *pInputName = NULL; /* our inputName currently is always "imuxsock", and this will hold it */ static int startIndexUxLocalSockets; /* process funix from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup @@ -312,7 +312,7 @@ CODESTARTwillRun /* we need to create the inputName property (only once during our lifetime) */ CHKiRet(prop.Construct(&pInputName)); - CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imudp"), sizeof("imudp") - 1)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); finalize_it: -- cgit v1.2.3 From 184d8cf6d27ea1753cc78b74b6d81a8433a6d59a Mon Sep 17 00:00:00 2001 From: Marc Schiffbauer Date: Mon, 8 Feb 2010 14:20:15 +0100 Subject: improved PostgreSQL database create script, provided doc on PostgreSQL setup Signed-off-by: Rainer Gerhards --- doc/src/rsyslog-pgsql.odt | Bin 0 -> 41755 bytes plugins/ompgsql/createDB.sql | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 doc/src/rsyslog-pgsql.odt diff --git a/doc/src/rsyslog-pgsql.odt b/doc/src/rsyslog-pgsql.odt new file mode 100644 index 00000000..5034c5fb Binary files /dev/null and b/doc/src/rsyslog-pgsql.odt differ diff --git a/plugins/ompgsql/createDB.sql b/plugins/ompgsql/createDB.sql index facdcaa0..2f72a0a6 100644 --- a/plugins/ompgsql/createDB.sql +++ b/plugins/ompgsql/createDB.sql @@ -1,4 +1,4 @@ -CREATE DATABASE "Syslog"; +CREATE DATABASE 'Syslog' WITH ENCODING 'SQL_ASCII'; \c Syslog; CREATE TABLE SystemEvents ( -- cgit v1.2.3 From c34e7d21146a3e6722bbf159a06750fdef23b4f8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Feb 2010 14:30:08 +0100 Subject: integrated new PostgreSQL doc converted to html, linked, etc... --- ChangeLog | 2 + doc/Makefile.am | 1 + doc/manual.html | 1 + doc/rsyslog_pgsql.html | 336 ++++++++++++++++++++++++++++++++++++++++++++++ doc/src/rsyslog-pgsql.odt | Bin 41755 -> 0 bytes doc/src/rsyslog_pgsql.odt | Bin 0 -> 41755 bytes 6 files changed, 340 insertions(+) create mode 100644 doc/rsyslog_pgsql.html delete mode 100644 doc/src/rsyslog-pgsql.odt create mode 100644 doc/src/rsyslog_pgsql.odt diff --git a/ChangeLog b/ChangeLog index bfcb1fa6..37a707f5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.5.8 [v4-beta] (rgerhards), 2010-01-?? +- enhanced doc for using PostgreSQL + Thanks to Marc Schiffbauer for the new/updated doc - bugfix: imuxsock incorrectly stated inputname "imudp" Thanks to Ryan Lynch for reporting this. - (slightly) enhanced support for FreeBSD by setting _PATH_MODDIR to diff --git a/doc/Makefile.am b/doc/Makefile.am index 3dfc8d3a..a1f192ee 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -19,6 +19,7 @@ html_files = \ rsyslog_conf.html \ rsyslog-example.conf \ rsyslog_mysql.html \ + rsyslog_pgsql.html \ rsyslog_packages.html \ rsyslog_high_database_rate.html \ rsyslog_php_syslog_ng.html \ diff --git a/doc/manual.html b/doc/manual.html index 785845b6..289d490b 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -55,6 +55,7 @@ generic syslog application design
  • using multiple rule sets in rsyslog
  • ssl-encrypting syslog with stunnel
  • writing syslog messages to MySQL (and other databases as well)
    • +
  • writing syslog messages to PostgreSQL (and other databases as well)
  • writing massive amounts of syslog messages to a database
  • reliable forwarding to a remote server
  • using diff --git a/doc/rsyslog_pgsql.html b/doc/rsyslog_pgsql.html new file mode 100644 index 00000000..dcb9dc3a --- /dev/null +++ b/doc/rsyslog_pgsql.html @@ -0,0 +1,336 @@ + + + + + + + + + + + + + + + + + +

    Writing syslog messages +to MySQL, PostgreSQL or any other supported Database

    +

    Written by Rainer +Gerhards with some additions by Marc +Schiffbauer (2008-02-28)

    +

    Abstract

    +

    In this paper, I describe +how to write syslog +messages to a MySQL +or PostgreSQL +database. Having syslog messages +in a database is often handy, especially when you intend to set up a +front-end for viewing them. This paper describes an approach with +rsyslogd, +an alternative enhanced syslog daemon natively supporting MySQL and +PostgreSQL. I describe the components needed to be installed and how +to configure them. Please note that as of this writing, rsyslog +supports a variety of databases. While this guide is still MySQL- and +PostgreSQL-focused, you can probably use it together with other ones +too. You just need to modify a few settings.

    +

    Background

    +

    In many cases, syslog data is simply +written to text files. This approach has some advantages, most +notably it is very fast and efficient. However, data stored in text +files is not readily accessible for real-time viewing and analysis. +To do that, the messages need to be in a database. There are various +ways to store syslog messages in a database. For example, some have +the syslogd write text files which are later feed via a separate +script into the database. Others have written scripts taking the data +(via a pipe) from a non-database-aware syslogd and store them as they +appear. Some others use database-aware syslogds and make them write +the data directly to the database. In this paper, I use that "direct +write" approach. I think it is superior, because the syslogd +itself knows the status of the database connection and thus can +handle it intelligently (well ... hopefully ;)). I use rsyslogd to +acomplish this, simply because I have initiated the rsyslog project +with database-awareness as one goal.

    +

    One word of caution: +while message storage in the database provides an excellent +foundation for interactive analysis, it comes at a cost. Database i/o +is considerably slower than text file i/o. As such, directly writing +to the database makes sense only if your message volume is low enough +to allow a) the syslogd, b) the network, and c) the database server +to catch up with it. Some time ago, I have written a paper on +optimizing +syslog server performance. While this paper +talks about Window-based solutions, the ideas in it are generic +enough to apply here, too. So it might be worth reading if you +anticipate medium high to high traffic. If you anticipate really high +traffic (or very large traffic spikes), you should seriously consider +forgetting about direct database writes - in my opinion, such a +situation needs either a very specialized system or a different +approach (the text-file-to-database approach might work better for +you in this case). +

    +

    Overall System Setup

    +

    In this paper, I concentrate on +the server side. If you are thinking about interactive syslog message +review, you probably want to centralize syslog. In such a scenario, +you have multiple machines (the so-called clients) send their data to +a central machine (called server in this context). While I expect +such a setup to be typical when you are interested in storing +messages in the database, I do not describe how to set it up. This is +beyond the scope of this paper. If you search a little, you will +probably find many good descriptions on how +to centralize syslog. If you do that, it might be a good idea to do +it securely, so you might also be interested in my paper on +ssl-encrypting +syslog message transfer.

    +

    No matter how the messages arrive at +the server, their processing is always the same. So you can use this +paper in combination with any description for centralized syslog +reporting.

    +

    As I already said, I use +rsyslogd on the server. It has intrinsic support for talking to the +supported databases. For obvious reasons, we also need an instance of +MySQL or PostgreSQL running. To keep us focused, the setup of the +database itself is also beyond the scope of this paper. I assume that +you have successfully installed the database and also have a +front-end at hand to work with it (for example, phpMyAdmin +or phpPgAdmin. +Please make sure that this is installed, actually working and you +have a basic understanding of how to handle it.

    +

    Setting up the system

    +

    You need to download and +install rsyslogd first. Obtain it from the rsyslog +site. Make sure that you disable stock +syslogd, otherwise you will experience some difficulties. On some +distributions  (Fedora 8 and above, for example), rsyslog may +already by the default syslogd, in which case you obviously do not +need to do anything specific. For many others, there are prebuild +packages available. If you use either, please make sure that you have +the required database plugins for your database available. It usually +is a separate package and typically not +installed by default.

    +

    It is important to understand +how rsyslogd talks to the database. In rsyslogd, there is the concept +of "templates". Basically, a template is a string that +includes some replacement characters, which are called "properties" +in rsyslog. Properties are accessed via the "Property +Replacer". Simply said, you access +properties by including their name between percent signs inside the +template. For example, if the syslog message is "Test", the +template "%msg%" would be expanded to "Test". +Rsyslogd supports sending template text as a SQL statement to the +database. As such, the template must be a valid SQL statement. There +is no limit in what the statement might be, but there are some +obvious and not so obvious choices. For example, a template "drop +table xxx" is possible, but does not make an awful lot of sense. +In practice, you will always use an "insert" statement +inside the template.

    +

    An example: if you would just like to +store the msg part of the full syslog message, you have probably +created a table "syslog" with a single column "message". +In such a case, a good template would be "insert into +syslog(message) values ('%msg%')". With the example above, that +would be expanded to "insert into syslog(message) +values('Test')". This expanded string is then sent to the +database. It's that easy, no special magic. The only thing you must +ensure is that your template expands to a proper SQL statement and +that this statement matches your database design.

    +

    Does that mean you need to +create database schema yourself and also must fully understand +rsyslogd's properties? No, that's not needed. Because we anticipated +that folks are probably more interested in getting things going +instead of designing them from scratch. So we have provided a default +schema as well as build-in support for it. This schema also offers an +additional benefit: rsyslog is part of Adiscon's +MonitorWare product +line (which includes open source and closed +source members). All of these tools share the same default schema and +know how to operate on it. For this reason, the default schema is +also called the "MonitorWare Schema". If you use it, you +can simply add phpLogCon, +a GPLed syslog web interface, to your system +and have instant interactive access to your database. So there are +some benefits in using the provided schema.

    +

    The schema definition is contained in +the file "createDB.sql". It comes with the rsyslog package +and one can be found for each supported database type (in the plugins +directory). Review it to check that the database name is acceptable +for you. Be sure to leave the table and field names unmodified, +because otherwise you need to customize rsyslogd's default sql +template, which we do not do in this paper. Then, run the script with +your favorite SQL client. Double-check that the table was +successfully created.

    +

    It is important to note that the +correct database encoding must be used so that the database will +accept strings independend of the string encoding. This is an +important part because it can not be guarantied that all syslog +messages will have a defined character encoding. This is especially +true if the rsyslog-Server will collect messages from different +clients and different products. +

    +

    For example PostgreSQL may refuse to +accept messages if you would set the database encoding to “UTF8” +while a client is sending invalid byte sequences for that encoding. +

    +

    Database support in rsyslog is +integrated via loadable plugin modules. To use the database +functionality, the database plugin must be enabled in the config file +BEFORE the first database table action is used. This is done by +placing the

    +
    $ModLoad ommysql
    +

    directive at the begining of /etc/rsyslog.conf for +MySQL and

    +
    $ModLoad ompgsql
    +

    for +PostgreSQL.

    +

    For +other databases, use their plugin name (e.g. omoracle).

    +

    Next, we need to tell rsyslogd to write data to +the database. As we use the default schema, we do NOT need to define +a template for this. We can use the hardcoded one (rsyslogd handles +the proper template linking). So all we need to do e.g. for MySQL is +add a simple selector line to /etc/rsyslog.conf:

    +
    *.*       +:ommysql:database-server,database-name,database-userid,database-password
    +

    Again, other databases have other selector names, +e.g. ":ompgsql:" instead of ":ommysql:". See the +output plugin's documentation for details.

    +

    In many cases, the database will run +on the local machine. In this case, you can simply use "127.0.0.1" +for database-server. This can be especially advisable, if you +do not need to expose the database to any process outside of the +local machine. In this case, you can simply bind it to 127.0.0.1, +which provides a quite secure setup. Of course, rsyslog also supports +remote database instances. In that case, use the remote server name +(e.g. mydb.example.com) or IP-address. The database-name by +default is "Syslog". If you have modified the default, use +your name here. Database-userid and -password are the +credentials used to connect to the database. As they are stored in +clear text in rsyslog.conf, that user should have only the least +possible privileges. It is sufficient to grant it INSERT privileges +to the systemevents table, only. As a side note, it is strongly +advisable to make the rsyslog.conf file readable by root only - if +you make it world-readable, everybody could obtain the password (and +eventually other vital information from it). In our example, let's +assume you have created a database user named "syslogwriter" +with a password of "topsecret" (just to say it bluntly: +such a password is NOT a good idea...). If your database is on the +local machine, your rsyslog.conf line might look like in this sample:

    +
    *.*       +:ommysql:127.0.0.1,Syslog,syslogwriter,topsecret
    +

    Save rsyslog.conf, restart rsyslogd - and you +should see syslog messages being stored in the "systemevents" +table!

    +

    The example line stores every message +to the database. Especially if you have a high traffic volume, you +will probably limit the amount of messages being logged. This is easy +to accomplish: the "write database" action is just a +regular selector line. As such, you can apply normal selector-line +filtering. If, for example, you are only interested in messages from +the mail subsystem, you can use the following selector line:

    +
    mail.*       :ommysql:127.0.0.1,syslog,syslogwriter,topsecret
    +

    Review the rsyslog.conf +documentation for details on selector lines and their filtering.

    +

    You have now completed +everything necessary to store syslog messages to the a database. +If you would like to try out a front-end, you might want to look at +phpLogCon, +which displays syslog data in a browser. As of this writing, +phpLogCon is not yet a powerful tool, but it's open source, so it +might be a starting point for your own solution.

    +

    On Reliability...

    +

    Rsyslogd writes syslog messages +directly to the database. This implies that the database must be +available at the time of message arrival. If the database is offline, +no space is left or something else goes wrong - rsyslogd can not +write the database record. If rsyslogd is unable to store a message, +it performs one retry. This is helpful if the database server was +restarted. In this case, the previous connection was broken but a +reconnect immediately succeeds. However, if the database is down for +an extended period of time, an immediate retry does not help.

    +

    Message loss in this scenario +can easily be prevented with rsyslog. All you need to do is run the +database writer in queued mode. This is now described in a generic +way and I do not intend to duplicate it here. So please be sure to +read "Handling +a massive syslog database insert rate with Rsyslog", +which describes the scenario and also includes configuration +examples.

    +

    Conclusion

    +

    With minimal effort, you can use +rsyslogd to write syslog messages to a database. You can even make it +absolutely fail-safe and protect it against database server downtime. +Once the messages are arrived there, you can interactively review and +analyze them. In practice, the messages are also stored in text files +for longer-term archival and the databases are cleared out after some +time (to avoid becoming too slow). If you expect an extremely high +syslog message volume, storing it in real-time to the database may +outperform your database server. In such cases, either filter out +some messages or used queued mode (which in general is recommended +with databases).

    +

    The method outlined in this paper +provides an easy to setup and maintain solution for most use cases.

    +

    Feedback Requested

    +

    I would appreciate feedback on this paper. If you +have additional ideas, comments or find bugs, please let +me know.

    +

    References and Additional Material

    + +

    Revision History

    +
      +

    • 2005-08-02 * + Rainer + Gerhards * initial version created +

      +

    • 2005-08-03 * + Rainer + Gerhards * added references to demo site +

      +

    • 2007-06-13 * + Rainer + Gerhards * removed demo site - was torn down because too + expensive for usage count +

      +

    • 2008-02-21 * + Rainer + Gerhards * updated reliability section, can now be done with + on-demand disk queues

      +

    • 2008-02-28 * Rainer + Gerhards * added info on other databases, updated syntax to more + recent one +

      +

    • 2010-01-29 * Marc Schiffbauer * added some + PostgreSQL stuff, made wording more database generic, fixed some + typos

      +
    +

    Copyright

    +

    Copyright (c) 2005-2010 Rainer +Gerhards, Marc Schiffbauer and Adiscon.

    +



    +

    + + \ No newline at end of file diff --git a/doc/src/rsyslog-pgsql.odt b/doc/src/rsyslog-pgsql.odt deleted file mode 100644 index 5034c5fb..00000000 Binary files a/doc/src/rsyslog-pgsql.odt and /dev/null differ diff --git a/doc/src/rsyslog_pgsql.odt b/doc/src/rsyslog_pgsql.odt new file mode 100644 index 00000000..5034c5fb Binary files /dev/null and b/doc/src/rsyslog_pgsql.odt differ -- cgit v1.2.3 From e1584b71f316b9ef2db58c8dbd2218f0b38962e7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Feb 2010 16:53:30 +0100 Subject: several bugfixes for the property replacer - bugfix: property replacer returned invalid parameters under some (unusual) conditions. In extreme cases, this could lead to garbled logs and/or a system failure. - bugfix: invalid length returned (often) when using regular expressions inside the property replacer - bugfix: submatch regex in property replacer did not honor "return 0 on no match" config case --- ChangeLog | 7 ++++ runtime/msg.c | 102 +++++++++++++++++++++++++++++++++------------------------- 2 files changed, 65 insertions(+), 44 deletions(-) diff --git a/ChangeLog b/ChangeLog index 37a707f5..bc5d5f53 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,13 @@ Version 4.5.8 [v4-beta] (rgerhards), 2010-01-?? - enhanced doc for using PostgreSQL Thanks to Marc Schiffbauer for the new/updated doc +- bugfix: property replacer returned invalid parameters under some (unusual) + conditions. In extreme cases, this could lead to garbled logs and/or + a system failure. +- bugfix: invalid length returned (often) when using regular expressions + inside the property replacer +- bugfix: submatch regex in property replacer did not honor "return 0 on + no match" config case - bugfix: imuxsock incorrectly stated inputname "imudp" Thanks to Ryan Lynch for reporting this. - (slightly) enhanced support for FreeBSD by setting _PATH_MODDIR to diff --git a/runtime/msg.c b/runtime/msg.c index b45775b6..8e3ad314 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2139,6 +2139,10 @@ static uchar *getNOW(eNOWType eNow) * be used in selector line processing. * rgerhards 2005-09-15 */ +/* a quick helper to save some writing: */ +#define RET_OUT_OF_MEMORY { *pbMustBeFreed = 0;\ + *pPropLen = sizeof("**OUT OF MEMORY**") - 1; \ + return(UCHAR_CONSTANT("**OUT OF MEMORY**"));} uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, propid_t propID, size_t *pPropLen, unsigned short *pbMustBeFreed) @@ -2200,8 +2204,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, case PROP_PRI_TEXT: pBuf = malloc(20 * sizeof(uchar)); if(pBuf == NULL) { - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; pRes = (uchar*)textpri((char*)pBuf, 20, getPRIi(pMsg)); @@ -2245,49 +2248,49 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, break; case PROP_SYS_NOW: if((pRes = getNOW(NOW_NOW)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_YEAR: if((pRes = getNOW(NOW_YEAR)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_MONTH: if((pRes = getNOW(NOW_MONTH)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_DAY: if((pRes = getNOW(NOW_DAY)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_HOUR: if((pRes = getNOW(NOW_HOUR)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_HHOUR: if((pRes = getNOW(NOW_HHOUR)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_QHOUR: if((pRes = getNOW(NOW_QHOUR)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_MINUTE: if((pRes = getNOW(NOW_MINUTE)) == NULL) { - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; @@ -2299,6 +2302,8 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, * error message unreadable. rgerhards, 2007-07-10 */ dbgprintf("invalid property id: '%d'\n", propID); + *pbMustBeFreed = 0; + *pPropLen = sizeof("**INVALID PROPERTY NAME**") - 1; return UCHAR_CONSTANT("**INVALID PROPERTY NAME**"); } @@ -2357,8 +2362,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } /* now copy */ memcpy(pBuf, pFld, iLen); @@ -2375,6 +2379,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); *pbMustBeFreed = 0; + *pPropLen = sizeof("**FIELD NOT FOUND**") - 1; return UCHAR_CONSTANT("**FIELD NOT FOUND**"); } } else if(pTpe->data.field.iFromPos != 0 || pTpe->data.field.iToPos != 0) { @@ -2403,8 +2408,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } pSb = pRes; if(iFrom) { @@ -2434,9 +2438,15 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else { /* Check for regular expressions */ if (pTpe->data.field.has_regex != 0) { - if (pTpe->data.field.has_regex == 2) + if (pTpe->data.field.has_regex == 2) { /* Could not compile regex before! */ + if (*pbMustBeFreed == 1) { + free(pRes); + *pbMustBeFreed = 0; + } + *pPropLen = sizeof("**NO MATCH** **BAD REGULAR EXPRESSION**") - 1; return UCHAR_CONSTANT("**NO MATCH** **BAD REGULAR EXPRESSION**"); + } dbgprintf("string to match for regex is: %s\n", pRes); @@ -2476,12 +2486,16 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) - return UCHAR_CONSTANT("**NO MATCH**"); - else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) - return UCHAR_CONSTANT("0"); - else - return UCHAR_CONSTANT(""); + if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) { + bufLen = sizeof("**NO MATCH**") - 1; + pRes = UCHAR_CONSTANT("**NO MATCH**"); + } else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) { + bufLen = 1; + pRes = UCHAR_CONSTANT("0"); + } else { + bufLen = 0; + pRes = UCHAR_CONSTANT(""); + } } } else { /* Match- but did it match the one we wanted? */ @@ -2492,10 +2506,16 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } - if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) - return UCHAR_CONSTANT("**NO MATCH**"); - else - return UCHAR_CONSTANT(""); + if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR) { + bufLen = sizeof("**NO MATCH**") - 1; + pRes = UCHAR_CONSTANT("**NO MATCH**"); + } else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO) { + bufLen = 1; + pRes = UCHAR_CONSTANT("0"); + } else { + bufLen = 0; + pRes = UCHAR_CONSTANT(""); + } } } /* OK, we have a usable match - we now need to malloc pB */ @@ -2509,13 +2529,12 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if (pB == NULL) { if (*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } /* Lets copy the matched substring to the buffer */ memcpy(pB, pRes + iOffs + pmatch[pTpe->data.field.iSubMatchToUse].rm_so, iLenBuf); - bufLen = iLenBuf - 1; + bufLen = iLenBuf; pB[iLenBuf] = '\0';/* terminate string, did not happen before */ if (*pbMustBeFreed == 1) @@ -2533,6 +2552,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, free(pRes); *pbMustBeFreed = 0; } + *pPropLen = sizeof("***REGEXP NOT AVAILABLE***") - 1; return UCHAR_CONSTANT("***REGEXP NOT AVAILABLE***"); } } @@ -2565,8 +2585,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } pSrc = pRes; while(*pSrc) { @@ -2612,8 +2631,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(!iscntrl((int) *pSrc)) @@ -2648,8 +2666,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(iscntrl((int) *pSrc)) @@ -2688,8 +2705,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } while(*pRes) { if(iscntrl((int) *pRes)) { @@ -2734,8 +2750,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(*pSrc != '/') @@ -2770,8 +2785,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } for(pSrc = pRes; *pSrc; pSrc++) { if(*pSrc == '/') @@ -2825,8 +2839,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* ok, original copy, need a private one */ pB = malloc((iLn + 1) * sizeof(uchar)); if(pB == NULL) { - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } memcpy(pB, pRes, iLn - 1); pRes = pB; @@ -2845,6 +2858,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pTpe->data.field.options.bCSV) { /* we need to obtain a private copy, as we need to at least add the double quotes */ int iBufLen; + int i; uchar *pBStart; uchar *pDst; uchar *pSrc; @@ -2856,10 +2870,10 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); - *pbMustBeFreed = 0; - return UCHAR_CONSTANT("**OUT OF MEMORY**"); + RET_OUT_OF_MEMORY; } pSrc = pRes; + i = 0; *pDst++ = '"'; /* starting quote */ while(*pSrc) { if(*pSrc == '"') -- cgit v1.2.3 From eff5630ec0944b9fe10a50be7f236ac714807be7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Feb 2010 17:28:58 +0100 Subject: enhanced test suite now CSV and regex extraction are being tested (would have caught the bug fixed with the last commit). --- tests/parsertest.sh | 12 ++++++++++-- tests/testsuites/parse3.conf | 8 ++++++++ tests/testsuites/reallife.parse3 | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 tests/testsuites/parse3.conf create mode 100644 tests/testsuites/reallife.parse3 diff --git a/tests/parsertest.sh b/tests/parsertest.sh index ef33256e..e0d35218 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,5 +1,13 @@ -echo TEST: parsertest.sh - various parser tests +echo TEST: \[parsertest.sh\]: various parser tests source $srcdir/diag.sh init source $srcdir/diag.sh nettester parse1 udp source $srcdir/diag.sh nettester parse1 tcp -source $srcdir/diag.sh init +source $srcdir/diag.sh nettester parse3 udp +source $srcdir/diag.sh nettester parse3 tcp + +echo \[parsertest.sh]: redoing tests in IPv4-only mode +source $srcdir/diag.sh nettester parse1 udp -4 +source $srcdir/diag.sh nettester parse1 tcp -4 +source $srcdir/diag.sh nettester parse3 udp -4 +source $srcdir/diag.sh nettester parse3 tcp -4 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/parse3.conf b/tests/testsuites/parse3.conf new file mode 100644 index 00000000..d5cf77d9 --- /dev/null +++ b/tests/testsuites/parse3.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9]+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" +*.* :omstdout:;output diff --git a/tests/testsuites/reallife.parse3 b/tests/testsuites/reallife.parse3 new file mode 100644 index 00000000..465635bd --- /dev/null +++ b/tests/testsuites/reallife.parse3 @@ -0,0 +1,15 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# This file contains a lot of real-life samples (of course mangled so +# that they can not be traced back to the original submitter). Note +# that IP addr 192.0.2.1 is specifically set aside for testing and +# documentation by IANA. +# rgerhards, 2009-10-19 +<175>Oct 16 2009 23:47:31 hostname tag This is a message +"2009-10-16T23:47:31+01:00", "hostname", "tag", "", "7", " This is a message" +# +<175>Oct 16 2009 23:47:31 hostname tag[1234] This is a message +"2009-10-16T23:47:31+01:00", "hostname", "tag", "1234", "7", " This is a message" -- cgit v1.2.3 From b5e44445f9b0ac229b7802a46df33d0f2c4a2cd8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Feb 2010 18:35:27 +0100 Subject: added one more test to testbench --- tests/parsertest.sh | 4 ++++ tests/testsuites/parse_invld_regex.conf | 8 ++++++++ tests/testsuites/samples.parse_invld_regex | 16 ++++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 tests/testsuites/parse_invld_regex.conf create mode 100644 tests/testsuites/samples.parse_invld_regex diff --git a/tests/parsertest.sh b/tests/parsertest.sh index e0d35218..5966a11b 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -4,10 +4,14 @@ source $srcdir/diag.sh nettester parse1 udp source $srcdir/diag.sh nettester parse1 tcp source $srcdir/diag.sh nettester parse3 udp source $srcdir/diag.sh nettester parse3 tcp +source $srcdir/diag.sh nettester parse_invld_regex udp +source $srcdir/diag.sh nettester parse_invld_regex tcp echo \[parsertest.sh]: redoing tests in IPv4-only mode source $srcdir/diag.sh nettester parse1 udp -4 source $srcdir/diag.sh nettester parse1 tcp -4 source $srcdir/diag.sh nettester parse3 udp -4 source $srcdir/diag.sh nettester parse3 tcp -4 +source $srcdir/diag.sh nettester parse_invld_regex udp -4 +source $srcdir/diag.sh nettester parse_invld_regex tcp -4 source $srcdir/diag.sh exit diff --git a/tests/testsuites/parse_invld_regex.conf b/tests/testsuites/parse_invld_regex.conf new file mode 100644 index 00000000..736aae59 --- /dev/null +++ b/tests/testsuites/parse_invld_regex.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" +*.* :omstdout:;output diff --git a/tests/testsuites/samples.parse_invld_regex b/tests/testsuites/samples.parse_invld_regex new file mode 100644 index 00000000..9ac2c3aa --- /dev/null +++ b/tests/testsuites/samples.parse_invld_regex @@ -0,0 +1,16 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# the actual message is not important. There is an error inside the conf +# file, and all messages will trigger the same problem. +# NOTE: it is correct that the "BAD REGULAR EXPRESSION" error message is +# *NOT* run through the rest of the propety replace, in specific through +# the CSV escaper. We do not do this because it could potentially lead +# to an obfuscated error message, and thus making problems hard to find. As +# this is a real error case, there is no problem in not obeying to the +# configured format. +# rgerhards, 2010-02-08 +<175>Feb 08 2008 23:47:31 hostname tag This is a message +"2008-02-08T23:47:31+01:00", "hostname", "tag", **NO MATCH** **BAD REGULAR EXPRESSION**, "7", " This is a message" -- cgit v1.2.3 From dd2e1a69060b9eabde255051e088bbd6bc6e02de Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Feb 2010 17:47:16 +0100 Subject: fixed a small memory leak during config file parsing --- template.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/template.c b/template.c index 8c13573b..9c4a2a39 100644 --- a/template.c +++ b/template.c @@ -563,8 +563,11 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) /* got the name */ cstrFinalize(pStrB); - if(propNameToID(pStrB, &pTpe->data.field.propid) != RS_RET_OK) + if(propNameToID(pStrB, &pTpe->data.field.propid) != RS_RET_OK) { + cstrDestruct(&pStrB); return 1; + } + cstrDestruct(&pStrB); /* no longer needed, now use ID */ /* Check frompos, if it has an R, then topos should be a regex */ if(*p == ':') { -- cgit v1.2.3 From b1a118172074019c0dae9d54db9fbc2ca4824592 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Feb 2010 17:51:23 +0100 Subject: minor doc fix: wrong default ... that clearly showed I could not do my math ;) --- doc/rsconf1_markmessageperiod.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rsconf1_markmessageperiod.html b/doc/rsconf1_markmessageperiod.html index 2c833339..a6486ba1 100644 --- a/doc/rsconf1_markmessageperiod.html +++ b/doc/rsconf1_markmessageperiod.html @@ -7,7 +7,7 @@

    $MarkMessagePeriod

    Type: specific to immark input module

    -

    Default: 1800 (20 minutes)

    +

    Default: 1200 (20 minutes)

    Description:

    This specifies when mark messages are to be written to output modules. The time specified is in seconds. Specifying 0 is possible and disables mark -- cgit v1.2.3 From 76b3f2cd875dca755f4beee8dbf525c23d9730cf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Feb 2010 15:26:27 +0100 Subject: preparing for 4.5.8 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- tests/Makefile.am | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index bc5d5f53..e9f7214f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.5.8 [v4-beta] (rgerhards), 2010-01-?? +Version 4.5.8 [v4-beta] (rgerhards), 2010-02-10 - enhanced doc for using PostgreSQL Thanks to Marc Schiffbauer for the new/updated doc - bugfix: property replacer returned invalid parameters under some (unusual) diff --git a/configure.ac b/configure.ac index 9a637d4d..fea6c0dd 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.5.7],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.5.8],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index 289d490b..f258686e 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 4.5.7 (v4-beta branch) of rsyslog. +

    This documentation is for version 4.5.8 (v4-beta branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might diff --git a/tests/Makefile.am b/tests/Makefile.am index 2026aec1..7adebfac 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -89,6 +89,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rfc5424-2.parse1 \ testsuites/rfc5424-3.parse1 \ testsuites/rfc5424-4.parse1 \ + testsuites/parse3.conf \ + testsuites/reallife.parse3 \ + testsuites/parse_invld_regex.conf \ + testsuites/samples.parse_invld_regex \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ testsuites/1.field1 \ -- cgit v1.2.3