diff options
| -rw-r--r-- | ChangeLog | 1 | ||||
| -rw-r--r-- | doc/mmpstrucdata.html | 23 | ||||
| -rw-r--r-- | plugins/mmpstrucdata/mmpstrucdata.c | 6 | ||||
| -rw-r--r-- | tests/Makefile.am | 10 | ||||
| -rw-r--r-- | tests/chkseq.c | 29 | ||||
| -rwxr-xr-x | tests/mmpstrucdata.sh | 12 | ||||
| -rwxr-xr-x | tests/rfc5424parser.sh | 12 | ||||
| -rw-r--r-- | tests/tcpflood.c | 28 | ||||
| -rw-r--r-- | tests/testsuites/mmpstrucdata.conf | 12 | ||||
| -rw-r--r-- | tests/testsuites/rfc5424parser.conf | 10 |
10 files changed, 130 insertions, 13 deletions
@@ -60,6 +60,7 @@ Version 7.5.7 [v7-devel] 2013-11-?? * queue.workerThreadMinimumMessage set to queue.size / num workers For queues with very low queue.maxSize (< 100), "emergency" defaults will be used. +- bugfix: mmpstrucdata generated inaccessible properties - bugfix: RainerScript optimizer did not optimize PRI filters things like "if $syslogfacility-text == "local3"" were not converted to PRIFILT. This was a regression introduced in 7.5.6. diff --git a/doc/mmpstrucdata.html b/doc/mmpstrucdata.html index b4003062..8197d94a 100644 --- a/doc/mmpstrucdata.html +++ b/doc/mmpstrucdata.html @@ -13,6 +13,7 @@ <p><b>Description</b>:</p> <p>The mmpstrucdata parses RFC5424 structured data into the message json variable tree. +The data parsed, if available, is stored under "jsonRoot!rfc5424-sd!...". <p> </p> <p><b>Module Configuration Parameters</b>:</p> @@ -33,6 +34,10 @@ Specifies into which json container the data shall be parsed to. <p><b>Caveats/Known Bugs:</b> <ul> <li>this module is currently experimental; feedback is appreciated +<li>property names are treated case-insensitive in rsyslog. As such, +RFC5424 names are treated case-insensitive as well. If such names +only differ in case (what is not recommended anyways), problems will +occur. <li>structured data with duplicate SD-IDs and SD-PARAMS is not properly processed </ul> @@ -48,6 +53,24 @@ template(name="jsondump" type="string" string="%msg%: %$!%\n") action(type="omfile" file="/path/to/log" template="jsondump") </textarea> +<p><b>A more practical one:</b> +<p>Take this example message (inspired by RFC5424 sample;)): +<p><code><34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][id@2 test="tast"] BOM'su root' failed for lonvick on /dev/pts/8</code> +<p>We apply this configuration: +<p><textarea rows="6" cols="120">module(load="mmpstrucdata") +action(type="mmpstrucdata") +template(name="sample2" type="string" + string="ALL: %$!%\nSD: %$!RFC5424-SD%\nIUT:%$!rfc5424-sd!exampleSDID@32473!iut%\nRAWMSG: %rawmsg%\n\n") +action(type="omfile" file="/path/to/log" template="sample2") +</textarea> +<p>This will output: +<p><code>ALL: { "rfc5424-sd": { "examplesdid@32473": { "iut": "3", "eventsource": "Application", "eventid": "1011" }, "id@2": { "test": "tast" } } }</br> +SD: { "examplesdid@32473": { "iut": "3", "eventsource": "Application", "eventid": "1011" }, "id@2": { "test": "tast" } }</br> +IUT:3</br> +RAWMSG: <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][id@2 test="tast"] BOM'su root' failed for lonvick on /dev/pts/8</code> +<p>As you can seem, you can address each of the individual items. Note that the +case of the RFC5424 parameter names has been converted to lower case. + <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the diff --git a/plugins/mmpstrucdata/mmpstrucdata.c b/plugins/mmpstrucdata/mmpstrucdata.c index e9e36b27..680ba92b 100644 --- a/plugins/mmpstrucdata/mmpstrucdata.c +++ b/plugins/mmpstrucdata/mmpstrucdata.c @@ -31,6 +31,7 @@ #include <errno.h> #include <unistd.h> #include <stdint.h> +#include <ctype.h> #include "conf.h" #include "syslogd-types.h" #include "srUtils.h" @@ -218,7 +219,8 @@ dbgprintf("DDDD: parseSD_NAME %s\n", sdbuf+*curridx); if( sdbuf[i] == '=' || sdbuf[i] == '"' || sdbuf[i] == ']' || sdbuf[i] == ' ') break; - namebuf[j] = sdbuf[i++]; + namebuf[j] = tolower(sdbuf[i]); + ++i; } namebuf[j] = '\0'; dbgprintf("DDDD: parseSD_NAME, NAME: '%s'\n", namebuf); @@ -349,7 +351,7 @@ dbgprintf("DDDD: json: '%s'\n", json_object_get_string(json)); if(jroot == NULL) { ABORT_FINALIZE(RS_RET_ERR); } - json_object_object_add(jroot, "RFC5424-SD", json); + json_object_object_add(jroot, "rfc5424-sd", json); msgAddJSON(pMsg, pData->jsonRoot, jroot); finalize_it: RETiRet; diff --git a/tests/Makefile.am b/tests/Makefile.am index a2548a68..5465774d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -7,6 +7,7 @@ TESTS = $(TESTRUNS) if ENABLE_IMDIAG TESTS += \ stop-localvar.sh \ + rfc5424parser.sh \ arrayqueue.sh \ global_vars.sh \ da-mainmsg-q.sh \ @@ -122,6 +123,11 @@ TESTS += \ imptcp_conndrop.sh endif +if ENABLE_MMPSTRUCDATA +TESTS += \ + mmpstrucdata.sh +endif + if ENABLE_GNUTLS # TODO: re-enable in newer version #TESTS += \ @@ -300,6 +306,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/stop-localvar.conf \ global_vars.sh \ testsuites/global_vars.conf \ + rfc5424parser.sh \ + testsuites/rfc5424parser.conf \ rs_optimizer_pri.sh \ testsuites/rs_optimizer_pr.conf \ rscript_prifilt.sh \ @@ -526,6 +534,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ mysql-asyn.sh \ mysql-asyn-vg.sh \ testsuites/mysql-asyn.conf \ + mmpstrucdata.sh \ + testsuites/mmpstrucdata.conf \ cfg.sh # TODO: re-enable diff --git a/tests/chkseq.c b/tests/chkseq.c index bea9f83a..bd8597e8 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -51,6 +51,7 @@ int main(int argc, char *argv[]) int reachedEOF; int edLen; /* length of extra data */ static char edBuf[500*1024]; /* buffer for extra data (pretty large to be on the save side...) */ + static char ioBuf[sizeof(edBuf)+1024]; char *file = NULL; while((opt = getopt(argc, argv, "e:f:ds:vE")) != EOF) { @@ -103,14 +104,22 @@ int main(int argc, char *argv[]) for(i = start ; i < end+1 ; ++i) { if(bHaveExtraData) { - scanfOK = fscanf(fp, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + } if(edLen != (int) strlen(edBuf)) { printf("extra data length specified %d, but actually is %ld in record %d\n", edLen, (long) strlen(edBuf), i); exit(1); } } else { - scanfOK = fscanf(fp, "%d\n", &val) == 1 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d\n", &val) == 1 ? 1 : 0; + } } if(!scanfOK) { printf("scanf error in index i=%d\n", i); @@ -132,9 +141,11 @@ int main(int argc, char *argv[]) exit(1); } - if(feof(fp)) { + int c = getc(fp); + if(c == EOF) { reachedEOF = 1; } else { + ungetc(c, fp); /* if duplicates are permitted, we need to do a final check if we have duplicates at the * end of file. */ @@ -142,14 +153,22 @@ int main(int argc, char *argv[]) i = end; while(!feof(fp)) { if(bHaveExtraData) { - scanfOK = fscanf(fp, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + } if(edLen != (int) strlen(edBuf)) { printf("extra data length specified %d, but actually is %ld in record %d\n", edLen, (long) strlen(edBuf), i); exit(1); } } else { - scanfOK = fscanf(fp, "%d\n", &val) == 1 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d\n", &val) == 1 ? 1 : 0; + } } if(val != i) { diff --git a/tests/mmpstrucdata.sh b/tests/mmpstrucdata.sh new file mode 100755 index 00000000..62b6ba96 --- /dev/null +++ b/tests/mmpstrucdata.sh @@ -0,0 +1,12 @@ +# This file is part of the rsyslog project, released under ASL 2.0 +# rgerhards, 2013-11-22 +echo =============================================================================== +echo \[mmpstrucdata.sh\]: testing mmpstrucdata +source $srcdir/diag.sh init +source $srcdir/diag.sh startup mmpstrucdata.conf +sleep 1 +source $srcdir/diag.sh tcpflood -m100 -y +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99 +source $srcdir/diag.sh exit diff --git a/tests/rfc5424parser.sh b/tests/rfc5424parser.sh new file mode 100755 index 00000000..3f5be497 --- /dev/null +++ b/tests/rfc5424parser.sh @@ -0,0 +1,12 @@ +# This file is part of the rsyslog project, released under ASL 2.0 +# rgerhards, 2013-11-22 +echo =============================================================================== +echo \[rfc5424parser.sh\]: testing mmpstrucdata +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rfc5424parser.conf +sleep 1 +source $srcdir/diag.sh tcpflood -m100 -y +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99 +source $srcdir/diag.sh exit diff --git a/tests/tcpflood.c b/tests/tcpflood.c index b3cef2e0..f17363f2 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -48,13 +48,14 @@ * -b number of messages within a batch (default: 100,000,000 millions) * -Y use multiple threads, one per connection (which means 1 if one only connection * is configured!) + * -y use RFC5424 style test message * -z private key file for TLS mode * -Z cert (public key) file for TLS mode * -L loglevel to use for GnuTLS troubleshooting (0-off to 10-all, 0 default) * * Part of the testbench for rsyslog. * - * Copyright 2009, 2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009, 2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -111,6 +112,7 @@ static int targetPort = 13514; static int numTargetPorts = 1; static int dynFileIDs = 0; static int extraDataLen = 0; /* amount of extra data to add to message */ +static int useRFC5424Format = 0; /* should the test message be in RFC5424 format? */ static int bRandomizeExtraData = 0; /* randomize amount of extra data added */ static int numMsgsToSend; /* number of messages to send */ static unsigned numConnections = 1; /* number of connections to create */ @@ -363,8 +365,14 @@ genMsg(char *buf, size_t maxBuf, int *pLenBuf, struct instdata *inst) snprintf(dynFileIDBuf, sizeof(dynFileIDBuf), "%d:", rand() % dynFileIDs); } if(extraDataLen == 0) { - *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%c", - msgPRI, dynFileIDBuf, msgNum, frameDelim); + if(useRFC5424Format) { + *pLenBuf = snprintf(buf, maxBuf, "<%s>1 2003-03-01T01:00:00.000Z mymachine.example.com tcpflood " + "- tag [tcpflood@32473 MSGNUM=\"%8.8d\"] msgnum:%s%8.8d:%c", + msgPRI, msgNum, dynFileIDBuf, msgNum, frameDelim); + } else { + *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%c", + msgPRI, dynFileIDBuf, msgNum, frameDelim); + } } else { if(bRandomizeExtraData) edLen = ((long) rand() + extraDataLen) % extraDataLen + 1; @@ -372,8 +380,14 @@ genMsg(char *buf, size_t maxBuf, int *pLenBuf, struct instdata *inst) edLen = extraDataLen; memset(extraData, 'X', edLen); extraData[edLen] = '\0'; - *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s%c", - msgPRI, dynFileIDBuf, msgNum, edLen, extraData, frameDelim); + if(useRFC5424Format) { + *pLenBuf = snprintf(buf, maxBuf, "<%s>1 2003-03-01T01:00:00.000Z mymachine.example.com tcpflood " + "- tag [tcpflood@32473 MSGNUM=\"%8.8d\"] msgnum:%s%8.8d:%c", + msgPRI, msgNum, dynFileIDBuf, msgNum, frameDelim); + } else { + *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s%c", + msgPRI, dynFileIDBuf, msgNum, edLen, extraData, frameDelim); + } } } else { /* use fixed message format from command line */ @@ -830,7 +844,7 @@ int main(int argc, char *argv[]) setvbuf(stdout, buf, _IONBF, 48); - while((opt = getopt(argc, argv, "b:ef:F:t:p:c:C:m:i:I:P:d:Dn:L:M:rsBR:S:T:XW:Yz:Z:")) != -1) { + while((opt = getopt(argc, argv, "b:ef:F:t:p:c:C:m:i:I:P:d:Dn:L:M:rsBR:S:T:XW:yYz:Z:")) != -1) { switch (opt) { case 'b': batchsize = atoll(optarg); break; @@ -908,6 +922,8 @@ int main(int argc, char *argv[]) break; case 'Y': runMultithreaded = 1; break; + case 'y': useRFC5424Format = 1; + break; case 'z': tlsKeyFile = optarg; break; case 'Z': tlsCertFile = optarg; diff --git a/tests/testsuites/mmpstrucdata.conf b/tests/testsuites/mmpstrucdata.conf new file mode 100644 index 00000000..fd18fd99 --- /dev/null +++ b/tests/testsuites/mmpstrucdata.conf @@ -0,0 +1,12 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/mmpstrucdata/.libs/mmpstrucdata") +module(load="../plugins/imtcp/.libs/imtcp") + +template(name="outfmt" type="string" string="%$!rfc5424-sd!tcpflood@32473!msgnum%\n") + +input(type="imtcp" port="13514") + +action(type="mmpstrucdata") +if $msg contains "msgnum" then + action(type="omfile" template="outfmt" file="rsyslog.out.log") diff --git a/tests/testsuites/rfc5424parser.conf b/tests/testsuites/rfc5424parser.conf new file mode 100644 index 00000000..cd90d120 --- /dev/null +++ b/tests/testsuites/rfc5424parser.conf @@ -0,0 +1,10 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/imtcp/.libs/imtcp") + +template(name="outfmt" type="string" string="%msg:F,58:2%\n") + +input(type="imtcp" port="13514") + +if $msg contains "msgnum" then + action(type="omfile" template="outfmt" file="rsyslog.out.log") |