Merge branch 'v3-stable' into beta

Conflicts:

	ChangeLog
	doc/manual.html

7 files changed, 118 insertions, 8 deletions

diff --git a/ChangeLog b/ChangeLog
index 1dded514..8509ad2a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -114,7 +114,19 @@ Version 3.21.0  [DEVEL] (rgerhards), 2008-07-18
 - imported all changes from 3.18.1 until today (some quite important,
   see below)
 ---------------------------------------------------------------------------
-Version 3.20.0 [BETA] (rgerhards), 2008-11-05
+Version 3.20.1 [v3-stable] (rgerhards), 2008-11-??
+- enhance: regex nomatch option "ZERO" has been added
+  This allows to return the string 0 if a regular expression is
+  not found. This is probably useful for storing numerical values into
+  database columns.
+- doc update: documented how to specify multiple property replacer
+  options + link to new online regex generator tool added
+- improved debug output for regular expressions inside property replacer
+  RE's seem to be a big trouble spot and I would like to have more
+  information inside the debug log. So I decided to add some additional
+  debug strings permanently.
+---------------------------------------------------------------------------
+Version 3.20.0 [v3-stable] (rgerhards), 2008-11-05
 - this is the inital release of the 3.19.x branch as a stable release
 - bugfix: double-free in pctp netstream driver. Thank to varmojfeko
   for the patch
diff --git a/doc/manual.html b/doc/manual.html
index 3e9d57a8..352d5257 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -34,7 +34,8 @@ the links below for the</b><br></p><ul>
 <li><a href="troubleshoot.html">troubleshooting rsyslog problems</a></li>
 <li><a href="rsyslog_conf.html">configuration file syntax (rsyslog.conf)</a></li>
 <li> <a href="property_replacer.html">property replacer, an important core component</a></li>
-<li>a commented <a href="sample.conf.html">sample rsyslog.conf</a></li>
+<li><a href="http://www.rsyslog.com/tool-regex">a regular expression checker/generator tool for rsyslog</a></li>
+<li>a commented <a href="sample.conf.html">sample rsyslog.conf</a> </li>
 <li><a href="bugs.html">rsyslog bug list</a></li>
 <li><a href="rsyslog_packages.html"> rsyslog packages</a></li>
 <li><a href="generic_design.html">backgrounder on
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index f666fb76..c2a0c0d2 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -240,10 +240,11 @@ that the first match is number 0, the second 1 and so on. Up to 10 matches
 (up to number 9) are supported. Please note that it would be more
 natural to have the match-number in front of submatch, but this would break 
 backward-compatibility. So the match-number must be specified after "nomatch".
-<p>nomatch is either "DFLT", "BLANK" or "FIELD" (all upper case!). It tells
+<p>nomatch is either "DFLT", "BLANK", ZERO or "FIELD" (all upper case!). It tells
 what to use if no match is found. With "DFLT", the strig "**NO MATCH**" is
 used. This was the only supported value up to rsyslog 3.19.5. With "BLANK"
-a blank text is used (""). Finally, "FIELD" uses the full property text
+a blank text is used (""). With "ZERO", "0" is used.
+Finally, "FIELD" uses the full property text
 instead of the expression. Some folks have requested that, so it seems
 to be useful.
 <p>The following is a sample of an ERE expression that takes the first
@@ -252,6 +253,13 @@ the full field if no match is found:
 <p>%msg:R,ERE,1,FIELD:for (vlan[0-9]*):--end%
 <p>and this takes the first submatch of the second match of said expression:
 <p>%msg:R,ERE,1,FIELD,1:for (vlan[0-9]*):--end%
+<p><b>Please note: there is also a
+<a href="http://www.rsyslog.com/tool-regex">rsyslog regular expression checker/generator</a>
+online tool available.</b> With that tool, you can check your regular expressions and
+also generate a valid property replacer sequence. Usage of this tool is recommended.
+Depending on the version offered, the tool may not cover all subleties that can
+be done with the property replacer. It concentrates on the most often used cases. So it
+is still useful to hand-craft expressions for demanding environments.
 <p><b>Also, extraction can be done based on so-called
 "fields"</b>. To do so, place a "F" into FromChar. A field in its
 current definition is anything that is delimited by a delimiter
@@ -390,6 +398,10 @@ Useful for secure pathname generation (with dynafiles).
 </tr>
 </tbody>
 </table>
+<p>To use multiple options, simply place them one after each other with a comma delmimiting
+them. For example "escape-cc,sp-if-no-1st-sp". If you use conflicting options together,
+the last one will override the previous one. For example, using "escape-cc,drop-cc" will
+use drop-cc and "drop-cc,escape-cc" will use escape-cc mode.
 <h2>Further Links</h2>
 <ul>
 <li>Article on "<a href="rsyslog_recording_pri.html">Recording
diff --git a/runtime/msg.c b/runtime/msg.c
index 9c2e3f17..3073fc5f 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -1958,7 +1958,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 				 * potential matches over the string.
 				 */
 				while(!bFound) {
-					if(regexp.regexec(&pTpe->data.field.re, pRes + iOffs, nmatch, pmatch, 0) == 0) {
+					int iREstat;
+					iREstat = regexp.regexec(&pTpe->data.field.re, pRes + iOffs, nmatch, pmatch, 0);
+					dbgprintf("regexec return is %d\n", iREstat);
+					if(iREstat == 0) {
 						if(pmatch[0].rm_so == -1) {
 							dbgprintf("oops ... start offset of successful regexec is -1\n");
 							break;
@@ -1966,6 +1969,8 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 						if(iTry == pTpe->data.field.iMatchToUse) {
 							bFound = 1;
 						} else {
+							dbgprintf("regex found at offset %d, new offset %d, tries %d\n",
+								  iOffs, iOffs + pmatch[0].rm_eo, iTry);
 							iOffs += pmatch[0].rm_eo;
 							++iTry;
 						}
@@ -1973,6 +1978,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 						break;
 					}
 				}
+				dbgprintf("regex: end search, found %d\n", bFound);
 				if(!bFound) {
 					/* we got no match! */
 					if(pTpe->data.field.nomatchAction != TPL_REGEX_NOMATCH_USE_WHOLE_FIELD) {
@@ -1982,6 +1988,8 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 						}
 						if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_DFLTSTR)
 							return "**NO MATCH**";
+						else if(pTpe->data.field.nomatchAction == TPL_REGEX_NOMATCH_USE_ZERO)
+							return "0";
 						else
 							return "";
 					}
diff --git a/template.c b/template.c
index 79167667..6fb7ba2b 100644
--- a/template.c
+++ b/template.c
@@ -558,13 +558,17 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 					pTpe->data.field.nomatchAction = TPL_REGEX_NOMATCH_USE_DFLTSTR;
 					p += 4; /* eat indicator sequence */
 				} else if(p[0] == 'B' && p[1] == 'L' && p[2] == 'A' && p[3] == 'N' && p[4] == 'K'
-				   && (p[5] == ',' || p[5] == ':')) {
+				          && (p[5] == ',' || p[5] == ':')) {
 					pTpe->data.field.nomatchAction = TPL_REGEX_NOMATCH_USE_BLANK;
 					p += 5; /* eat indicator sequence */
 				} else if(p[0] == 'F' && p[1] == 'I' && p[2] == 'E' && p[3] == 'L' && p[4] == 'D'
-				   && (p[5] == ',' || p[5] == ':')) {
+				          && (p[5] == ',' || p[5] == ':')) {
 					pTpe->data.field.nomatchAction = TPL_REGEX_NOMATCH_USE_WHOLE_FIELD;
 					p += 5; /* eat indicator sequence */
+				} else if(p[0] == 'Z' && p[1] == 'E' && p[2] == 'R' && p[3] == 'O'
+				          && (p[4] == ',' || p[4] == ':')) {
+					pTpe->data.field.nomatchAction = TPL_REGEX_NOMATCH_USE_ZERO;
+					p += 4; /* eat indicator sequence */
 				} else if(p[0] == ',') { /* empty, use default */
 					pTpe->data.field.nomatchAction = TPL_REGEX_NOMATCH_USE_DFLTSTR;
 					 /* do NOT eat indicator sequence, as this was already eaten - the 
diff --git a/template.h b/template.h
index 15b700b3..04137b09 100644
--- a/template.h
+++ b/template.h
@@ -78,7 +78,8 @@ struct templateEntry {
 			enum {
 				TPL_REGEX_NOMATCH_USE_DFLTSTR = 0, /* use the (old style) default "**NO MATCH**" string */
 				TPL_REGEX_NOMATCH_USE_BLANK = 1, /* use a blank string */
-				TPL_REGEX_NOMATCH_USE_WHOLE_FIELD = 2 /* use the full field contents that we were searching in*/
+				TPL_REGEX_NOMATCH_USE_WHOLE_FIELD = 2, /* use the full field contents that we were searching in*/
+				TPL_REGEX_NOMATCH_USE_ZERO = 3 /* use  0 (useful for numerical values) */
 			}  nomatchAction;	/**< what to do if we do not have a match? */
 			
 #endif
diff --git a/tools/regexp.c b/tools/regexp.c
new file mode 100644
index 00000000..c8e4c681
--- /dev/null
+++ b/tools/regexp.c
@@ -0,0 +1,72 @@
+/* A simple regular expression checker for rsyslog test and debug.
+ * Regular expressions have shown to turn out to be a hot support topic. 
+ * While I have done an online tool at http://www.rsyslog.com/tool-regex
+ * there are still some situations where one wants to check against the
+ * actual clib api calls. This is what this small test program does,
+ * it takes its command line arguments (re first, then sample data) and
+ * pushes them into the API and then shows the result. This should be
+ * considered the ultimate reference for any questions arising.
+ *
+ * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <regex.h>
+
+int main(int argc, char *argv[])
+{
+	regex_t preg;
+	size_t nmatch = 10;
+	regmatch_t pmatch[10];
+	char *pstr;
+	int i;
+
+	if(argc != 3) {
+		fprintf(stderr, "usage: regex regexp sample-data\n");
+		exit(1);
+	}
+
+	pstr = strdup(argv[2]); /* get working copy */
+
+	i = regcomp(&preg, argv[1], REG_EXTENDED);
+	printf("regcomp returns %d\n", i);
+	i = regexec(&preg, pstr, nmatch, pmatch, 0);
+	printf("regexec returns %d\n", i);
+	if(i == REG_NOMATCH) {
+		printf("found no match!\n");
+		return 1;
+	}
+
+	printf("returned substrings:\n");
+	for(i = 0 ; i < 10 ; i++) {
+		printf("%d: so %d, eo %d", i, pmatch[i].rm_so, pmatch[i].rm_eo);
+		if(pmatch[i].rm_so != -1) {
+			int j;
+			printf(", text: '");
+			for(j = pmatch[i].rm_so ; j < pmatch[i].rm_eo ; ++j)
+				putchar(pstr[j]);
+			putchar('\'');
+		}
+		putchar('\n');
+	}
+	return 0;
+}