From dfd541a1224a2ed407010e9924b6eab66c4643ba Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 28 Aug 2012 09:44:35 +0200
Subject: doc: new template system, some more output modules

added base doc for new config language
---
 doc/omfile.html                 |   2 +-
 doc/omsnmp.html                 | 148 ++++++++++++++++++++++++
 doc/property_replacer.html      | 246 ++++++++++++++++++++++++++++++++++++++--
 doc/rsyslog_conf_templates.html |  59 ++++++++--
 4 files changed, 434 insertions(+), 21 deletions(-)

diff --git a/doc/omfile.html b/doc/omfile.html
index bdd1ebc6..23ecc034 100644
--- a/doc/omfile.html
+++ b/doc/omfile.html
@@ -84,7 +84,7 @@
 *.* action(type="omfile" 
 DirCreateMode="0700"
 FileCreateMode="0644"
-File= "/var/log/messages")
+File="/var/log/messages")
 </textarea>
 
 <br><br>
diff --git a/doc/omsnmp.html b/doc/omsnmp.html
index b38a594f..202bb5bb 100644
--- a/doc/omsnmp.html
+++ b/doc/omsnmp.html
@@ -21,6 +21,153 @@ developer (headers) package installed. </p>
 <p>%omsnmp% without any further parameters.</p>
 <p>&nbsp;</p>
 <p><b>Configuration Directives</b>:</p>
+<ul>
+	<li><strong>transport </strong>(This parameter is optional, the 
+	default value is "udp")<br>
+	<br>
+	Defines the transport type you wish to use. Technically we can support all 
+	transport types which are supported by NET-SNMP. <br>
+	To name a few possible values: <br>
+	<br>
+	udp, tcp, udp6, tcp6, icmp, icmp6 ...<br>
+	<br>
+	Example: <strong>transport udp<br>
+	</strong></li>
+	<li><strong>server</strong><br>
+	<br>
+	This can be a hostname or ip address, and is our snmp target host. This 
+	parameter is required, if the snmptarget is not defined, nothing will be 
+	send. <br>
+	<br>
+	Example: <strong>server server.domain.xxx</strong><br>
+	</li>
+	<li><strong>port </strong>(This parameter is optional, the 
+	default value is "162")<br>
+	<br>
+	The port which will be used, common values are port 162 or 161. <br>
+	<br>
+	Example: <strong>port 162</strong><br>
+	</li>
+	<li><strong>version </strong>(This parameter is optional, the 
+	default value is "1")<br>
+	<br>
+	There can only be two choices for this parameter for now. <br>
+	0 means SNMPv1 will be used.<br>
+	1 means SNMPv2c will be used. <br>
+	Any other value will default to 1. <br>
+	<br>
+	Example: <strong>version 1</strong><br>
+	</li>
+	<li><strong>community </strong>(This parameter is optional, the 
+	default value is "public")<br>
+	<br>
+	This sets the used SNMP Community.<br>
+	<br>
+	Example:<strong> community public<br>
+	</strong><br>
+	</li>
+	<li><strong>trapoid </strong>(This parameter is 
+	optional, the default value is "1.3.6.1.4.1.19406.1.2.1&quot; which means 
+	&quot;ADISCON-MONITORWARE-MIB::syslogtrap&quot;)<br>
+	This configuration parameter is used for <strong>SNMPv2</strong> only.<br>
+	<br>
+	This is the OID which defines the trap-type, or notifcation-type rsyslog 
+	uses to send the trap. <br>
+	In order to decode this OID, you will need to have the 
+	ADISCON-MONITORWARE-MIB and ADISCON-MIB mibs installed on the receiver side. Downloads of these mib files 
+	can be found here: <br>
+	<a href="http://www.adiscon.org/download/ADISCON-MIB.txt">
+	http://www.adiscon.org/download/ADISCON-MIB.txt</a><br>
+	<a href="http://www.adiscon.org/download/ADISCON-MONITORWARE-MIB.txt">
+	http://www.adiscon.org/download/ADISCON-MONITORWARE-MIB.txt</a><br>
+	<br>
+	Thanks to the net-snmp 
+	mailinglist for the help and the recommendations ;).<br>
+	<br>
+	Example: <strong>trapoid 1.3.6.1.4.1.19406.1.2.1<br>
+	</strong>If you have this MIBS installed, you can also configured with the 
+	OID Name: <strong>trapoid ADISCON-MONITORWARE-MIB::syslogtrap<br>
+	</strong>
+	</li>
+	<li><strong>messageoid </strong>(This parameter is 
+	optional, the default value is "1.3.6.1.4.1.19406.1.1.2.1&quot; which means 
+	&quot;ADISCON-MONITORWARE-MIB::syslogMsg&quot;)<br>
+	<br>
+	This OID will be used as a variable, type &quot;OCTET STRING&quot;. This variable will 
+	contain up to 255 characters of the original syslog message including syslog header. It is recommend to 
+	use the default OID. <br>
+	In order to decode this OID, you will need to have the 
+	ADISCON-MONITORWARE-MIB and ADISCON-MIB mibs installed on the receiver side. 
+	To download these custom mibs, see the description of <strong>$actionsnmptrapoid.
+	</strong><br>
+	<br>
+	Example: <strong>messageoid 1.3.6.1.4.1.19406.1.1.2.1<br>
+	</strong>If you have this MIBS installed, you can also configured with the 
+	OID Name: <strong>messageoid 
+	ADISCON-MONITORWARE-MIB::syslogMsg<br>
+	</strong><br>
+	</li>
+	<li><strong>enterpriseoid </strong>(This parameter is optional, 
+	the default value is "1.3.6.1.4.1.3.1.1" which means "enterprises.cmu.1.1")<br>
+	<br>
+	Customize this value if needed. I recommend to use the default value unless 
+	you require to use a different OID. <br>
+	This configuration parameter is used for <strong>SNMPv1</strong> only. It 
+	has no effect if <strong>SNMPv2</strong> is used. <br>
+	<br>
+	Example: <strong>enterpriseoid 1.3.6.1.4.1.3.1.1 <br>
+	</strong><br>
+	</li>
+	<li><strong>specifictype </strong>(This parameter is optional, 
+	the default value is "0")<strong> </strong><br>
+	<br>
+	This is the specific trap number. This configuration parameter is used for
+	<strong>SNMPv1</strong> only. It has no effect if <strong>SNMPv2</strong> is 
+	used. <br>
+	<br>
+	Example: <strong>specifictype 0<br>
+	</strong><br>
+	</li>
+	<li><strong>traptype</strong> (This parameter is optional, the 
+	default value is "6" which means SNMP_TRAP_ENTERPRISESPECIFIC) <br>
+	<br>
+	There are only 7 Possible trap types defined which can be used here. These 
+	trap types are: <br>
+	0 = SNMP_TRAP_COLDSTART<br>
+	1 = SNMP_TRAP_WARMSTART<br>
+	2 = SNMP_TRAP_LINKDOWN<br>
+	3 = SNMP_TRAP_LINKUP<br>
+	4 = SNMP_TRAP_AUTHFAIL<br>
+	5 = SNMP_TRAP_EGPNEIGHBORLOSS<br>
+	6 = SNMP_TRAP_ENTERPRISESPECIFIC<br>
+	<br>
+	Any other value will default to 6 automatically. This configuration 
+	parameter is used for <strong>SNMPv1</strong> only. It has no effect if
+	<strong>SNMPv2</strong> is used. <br>
+	<br>
+	Example: <strong>traptype 6</strong><br>
+	</li>
+	<li><strong>template </strong>[templateName]<strong> </strong><br>
+	<br>
+	sets a new default template for file actions.
+	</li>
+</ul>
+<p>&nbsp;</p>
+<p><b>Caveats/Known Bugs:</b></p><ul><li>In order to decode the custom OIDs, you 
+	will need to have the adiscon mibs installed. </li></ul>
+<p><b>Sample:</b></p>
+<p>The following commands send every message as a snmp trap.</p>
+<textarea rows="10" cols="60">Module (path="omsnmp")
+*.* action( type="omsnmp"
+transport="udp"
+target="localhost"
+targetport="162"
+version="1"
+community="public")
+
+</textarea>
+
+<p><b>Legacy Configuration Directives</b>:</p>
 <ul>
 	<li><strong>$actionsnmptransport </strong>(This parameter is optional, the 
 	default value is "udp")<br>
@@ -164,6 +311,7 @@ $actionsnmpcommunity public
 *.* :omsnmp:
 </textarea>
 
+
 <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual 
 index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index 86a07474..c6464a3b 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -236,7 +236,7 @@ or none if no rule with this field did match.
 <p>Properties starting with a $-sign are so-called system
 properties. These do NOT stem from the message but are rather
 internally-generated.</p>
-<h2>Character Positions</h2>
+<h2>Legacy Character Positions</h2>
 <p><b><code>FromChar</code></b> and <b><code>toChar</code></b>
 are used to build substrings. They specify the offset within the string
 that should be copied. Offset counting starts at 1, so if you need to
@@ -304,15 +304,6 @@ fields in the property is requested. The field number must be placed in
 the "ToChar" parameter. An example where the 3rd field (delimited by
 TAB) from the msg property is extracted is as follows: "%msg:F:3%". The
 same example with semicolon as delimiter is "%msg:F,59:3%".</p>
-<p>The use of fields does not permit to select substrings, what is rather
-unfortunate. To solve this issue, starting with 6.3.9, fromPos and toPos
-can be specified for strings as well. However, the syntax is quite ugly, but
-it was the only way to integrate this functonality into the already-existing
-system. To do so, use ",fromPos" and ",toPos" during field extraction.
-Let's assume you want to extract the substring from position 5 to 9 in the previous
-example. Then, the syntax is as follows: "%msg:F,59,5:3,9%". As you can see,
-"F,59" means field-mode, with semicolon delimiter and ",5" means starting 
-at position 5. Then "3,9" means field 3 and string extraction to position 9.
 <p>Please note that the special characters "F" and "R" are
 case-sensitive. Only upper case works, lower case will return an error.
 There are no white spaces permitted inside the sequence (that will lead
@@ -342,6 +333,239 @@ It is modeled after perl compatible regular expressions.
 
 <h2>Property Options</h2>
 <b><code>property options</code></b> are
+case-insensitive. They are available as of version 6.5.0.
+Currently, the following options are defined:
+<p></p>
+<table>
+<tbody>
+<tr>
+<td><b>Name</b></td>
+<td>New format. Name of the template / property / constant.</td>
+</tr>
+<tr>
+<td><b>Outname</b></td>
+<td>This field permits to specify a field name for structured-data emitting property replacer options.
+If used for a constant a template with line style, unpredictable behaviour can occur.</td>
+</tr>
+<tr>
+<td><b>CaseConversion</b></td>
+<td>New format. Additional values below.</td>
+</tr>
+<tr>
+<td>upper</td>
+<td>convert property to lowercase only</td>
+</tr>
+<tr>
+<td>lower</td>
+<td>convert property text to uppercase only</td>
+</tr>
+<tr>
+<td><b>DateFormat</b></td>
+<td>New format, additional parameter is needed. See below.</td>
+</tr>
+<tr>
+<td>mysql</td>
+<td>format as mysql date</td>
+</tr>
+<tr>
+<td>pgsql</td>
+<td>format as pgsql date</td>
+</tr>
+<tr>
+<td>rfc3164</td>
+<td>format as RFC 3164 date</td>
+</tr>
+<tr>
+<tr>
+<td valign="top">rfc3164-buggyday</td>
+<td>similar to date-rfc3164, but emulates a common coding error: RFC 3164 demands
+that a space is written for single-digit days. With this option, a zero is
+written instead. This format seems to be used by syslog-ng and the
+date-rfc3164-buggyday option can be used in migration scenarios where otherwise
+lots of scripts would need to be adjusted. It is recommended <i>not</i> to use this
+option when forwarding to remote hosts - they may treat the date as invalid
+(especially when parsing strictly according to RFC 3164).</td>
+<br><i>This feature was introduced in rsyslog 4.6.2 and v4 versions above and
+5.5.3 and all versions above.</i>
+</tr>
+<tr>
+<td>rfc3339</td>
+<td>format as RFC 3339 date</td>
+</tr>
+<tr>
+<td>unixtimestamp</td>
+<td>format as unix timestamp (seconds since epoch)</td>
+</tr>
+<tr>
+<td>subseconds</td>
+<td>just the subseconds of a timestamp (always 0 for a low precision timestamp)</td>
+</tr>
+<tr>
+<td><b>ControlCharacters</b></td>
+<td>Option values for how to process control characters</td>
+</tr>
+<tr>
+<td valign="top">escape</td>
+<td>replace control characters (ASCII value 127 and values
+less then 32) with an escape sequence. The sequnce is
+"#&lt;charval&gt;" where charval is the 3-digit decimal value
+of the control character. For example, a tabulator would be replaced by
+"#009".<br>
+Note: using this option requires that <a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a>
+is set to off.</td>
+</tr>
+<tr>
+<td valign="top">space</td>
+<td>replace control characters by spaces<br>
+Note: using this option requires that <a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a>
+is set to off.</td>
+</tr>
+<tr>
+<td valign="top">drop</td>
+<td>drop control characters - the resulting string will
+neither contain control characters, escape sequences nor any other
+replacement character like space.<br>
+Note: using this option requires that <a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a>
+is set to off.</td>
+</tr>
+<tr>
+<td><b>SecurePath</b></td>
+<td>Option values for securing path templates.</td>
+</tr>
+<tr>
+<td valign="top">drop</td>
+<td>Drops slashes inside the field (e.g. "a/b" becomes "ab").
+Useful for secure pathname generation (with dynafiles).
+</td>
+</tr>
+<tr>
+<td valign="top">replace</td>
+<td>Replace slashes inside the field by an underscore. (e.g. "a/b" becomes "a_b").
+Useful for secure pathname generation (with dynafiles).
+</td>
+</tr>
+<tr>
+<td><b>Format</b></td>
+<td>Option values for the general output format.</td>
+</tr>
+<tr>
+<td>json</td>
+<td>encode the value so that it can be used inside a JSON field. This means
+that several characters (according to the JSON spec) are being escaped, for 
+example US-ASCII LF is replaced by "\n".
+The json option cannot be used together with either jsonf or csv options.
+</td>
+</tr>
+<tr>
+<td>jsonf</td>
+<td><i>(available in 6.3.9+)</i>
+This signifies that the property should be expressed as a json <b>f</b>ield.
+That means not only the property is written, but rather a complete json field in
+the format<br>
+"fieldname"="value"</b>
+where "filedname" is the assigend field name (or the property name if none was assigned)
+and value is the end result of property replacer operation. Note that value supports
+all property replacer options, like substrings, case converson and the like.
+Values are properly json-escaped. However, field names are (currently) not. It is
+expected that proper field names are configured.
+The jsonf option cannot be used together with either json or csv options.
+</td>
+</tr>
+<tr>
+<td valign="top">csv</td>
+<td>formats the resulting field (after all modifications) in CSV format
+as specified in <a href="http://www.ietf.org/rfc/rfc4180.txt">RFC 4180</a>.
+Rsyslog will always use double quotes. Note that in order to have full CSV-formatted
+text, you need to define a proper template. An example is this one:
+<br>$template csvline,"%syslogtag:::csv%,%msg:::csv%"
+<br>Most importantly, you need to provide the commas between the fields
+inside the template.
+The csv option cannot be used together with either json or jsonf options.
+<br><i>This feature was introduced in rsyslog 4.1.6.</i>
+</td>
+</tr>
+<tr>
+<td><b>droplastlf</b></td>
+<td>The last LF in the message (if any), is dropped.
+Especially useful for PIX.</td>
+</tr>
+<tr>
+<td valign="top"><b>spifno1stsp</b></td>
+<td>This option looks scary and should probably not be used by a user. For any field
+given, it returns either a single space character or no character at all. Field content
+is never returned. A space is returned if (and only if) the first character of the
+field's content is NOT a space. This option is kind of a hack to solve a problem rooted
+in RFC 3164: 3164 specifies no delimiter between the syslog tag sequence and the actual
+message text. Almost all implementation in fact delemit the two by a space. As of
+RFC 3164, this space is part of the message text itself. This leads to a problem when
+building the message (e.g. when writing to disk or forwarding). Should a delimiting
+space be included if the message does not start with one? If not, the tag is immediately
+followed by another non-space character, which can lead some log parsers to misinterpret
+what is the tag and what the message. The problem finally surfaced when the klog module
+was restructured and the tag correctly written. It exists with other message sources,
+too. The solution was the introduction of this special property replacer option. Now,
+the default template can contain a conditional space, which exists only if the
+message does not start with one. While this does not solve all issues, it should
+work good enough in the far majority of all cases. If you read this text and have
+no idea of what it is talking about - relax: this is a good indication you will never
+need this option. Simply forget about it ;)
+</td>
+</tr>
+<tr>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><b>New character position</b></td>
+<td>In addition to the above mentioned Character Positions in the legacy format,
+positions can be determined by specifying the correct options for the properties.
+Again, this is mostly for using the list format.</td>
+</tr>
+<tr>
+<td>position.From</td>
+<td>Character position in the property to start from.</td>
+</tr>
+<tr>
+<td>position.To</td>
+<td>Character position that determines the end for extraction. If the value is "$"
+then the end of the string will be used.</td>
+</tr>
+<tr>
+<td>field.Number</td>
+<td>The number of the field, which should be used for the search operation with Regex.</td>
+</tr>
+<tr>
+<td>field.Delimiter</td>
+<td>The Character that should delimit a field. Example: ",". Everything in a
+property until this character is considered a field.</td>
+</tr>
+<tr>
+<td>regex.Expression</td>
+<td>Value to be compared to property.</td>
+</tr>
+<tr>
+<td>regex.Type</td>
+<td>Values BRE or ERE</td>
+</tr>
+<tr>
+<td>regex.NoMatchMode</td>
+<td>DFLT, BLANK, ZERO, FIELD</td>
+</tr>
+<tr>
+<td>regex.Match</td>
+<td>Match to use.</td>
+</tr>
+<tr>
+<td>regex.Submatch</td>
+<td>Submatch to use. Values 0-9 whereas 0 = All</td>
+</tr>
+</tbody>
+</table>
+
+
+
+<h2>Legacy Property Options</h2>
+<b><code>property options</code></b> are
 case-insensitive. Currently, the following options are defined:
 <p></p>
 <table>
@@ -368,7 +592,7 @@ The json option cannot be used together with either jsonf or csv options.
 This signifies that the property should be expressed as a json <b>f</b>ield.
 That means not only the property is written, but rather a complete json field in
 the format<br>
-"fieldname":"value"</b>
+"fieldname"="value"</b>
 where "filedname" is the assigend field name (or the property name if none was assigned)
 and value is the end result of property replacer operation. Note that value supports
 all property replacer options, like substrings, case converson and the like.
diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html
index bd0b3253..b97f6609 100644
--- a/doc/rsyslog_conf_templates.html
+++ b/doc/rsyslog_conf_templates.html
@@ -33,26 +33,31 @@ string generator module, you need to know how to call it. Each such module has a
 which you need to know (look it up in the module doc or ask the developer). Let's assume
 that "mystrgen" is the module name. Then you can define a template for that strgen
 in the following way:
+
+<blockquote><code>template(name="MyTemplateName" type="plugin" string="mystrgen")</code></blockquote>
+<p>Legacy example:</p>
 <blockquote><code>$template MyTemplateName,=mystrgen</code></blockquote>
 (Of course, you must have first loaded the module via $ModLoad).
-<p>The important part is the equal sign: it tells the rsyslog config parser that
+<p>The important part is the equal sign in the legacy format: it tells the rsyslog config parser that
 no string follows but a strgen module name.
 <p>There are no additional parameters but the module name supported. This is because
 there is no way to customize anything inside such a "template" other than by
 modifying the code of the string generator.
 
 <p>So for most use cases, string-generator module based templates are <b>not</b>
-the route to take. Usually, us use <b>string based templates</b> instead.
+the route to take. Usually, we use <b>string based templates</b> instead.
 This is what the rest of the documentation now talks about.
 
 <p>A template consists of a template directive, a name, the
 actual template text and optional options. A sample is:</p>
+<blockquote><code>template(name="MyTemplateName" type="string" string="Example: Text %property% some more text\n" options)</code></blockquote>
+<p>Legacy example:</p>
 <blockquote><code>$template MyTemplateName,"\7Text
 %property% some more text\n",&lt;options&gt;</code></blockquote>
-<p>The "$template" is the template directive. It tells rsyslog
+<p>The "template" (legacy: $template) is the template directive. It tells rsyslog
 that this line contains a template. "MyTemplateName" is the template
 name. All
-other config lines refer to this name. The text within quotes is the
+other config lines refer to this name. The text within "string" is the
 actual template text. The backslash is an escape character, much as it
 is in C. It does all these "cool" things. For example, \7 rings the
 bell (this is an ASCII value), \n is a new line. C programmers and perl
@@ -69,24 +74,30 @@ on this is below, on some lines of the property replacer.<br>
 <br>
 The &lt;options&gt; part is optional. It carries options
 influencing the template as whole. See details below. Be sure NOT to
-mistake template options with property options - the later ones are
+mistake template options with property options - the latter ones are
 processed by the property replacer and apply to a SINGLE property, only
 (and not the whole template).<br>
 <br>
 Template options are case-insensitive. Currently defined are: </p>
-<p><b>sql</b> - format the string suitable for a SQL
+<p><b>option.sql</b> - format the string suitable for a SQL
 statement in MySQL format. This will replace single quotes ("'") and
 the backslash character by their backslash-escaped counterpart ("\'"
 and "\\") inside each field. Please note that in MySQL configuration,
 the <code class="literal">NO_BACKSLASH_ESCAPES</code>
 mode must be turned off for this format to work (this is the default).</p>
-<p><b>stdsql</b> - format the string suitable for a
+<p><b>option.stdsql</b> - format the string suitable for a
 SQL statement that is to be sent to a standards-compliant sql server.
 This will replace single quotes ("'") by two single quotes ("''")
 inside each field. You must use stdsql together with MySQL if in MySQL
 configuration the
 <code class="literal">NO_BACKSLASH_ESCAPES</code> is
 turned on.</p>
+<p><b>option.json</b> - format the string suitable for a
+json statement.
+This will replace single quotes ("'") by two single quotes ("''")
+inside each field.</p>
+<p>At no time, multiple template option should be used. This can cause
+unpredictable behaviour and is against all logic.</p>
 <p>Either the <b>sql</b> or <b>stdsql</b>&nbsp;
 option <b>must</b> be specified when a template is used
 for writing to a database, otherwise injection might occur. Please note
@@ -120,11 +131,41 @@ $template TraditionalFormat,"%timegenerated% %HOSTNAME% %syslogtag%%msg%\n"<br>
 <br>
 Properties can be accessed by the <a href="property_replacer.html">property
 replacer</a> (see there for details).</p>
-<p><b>Please note that templates can also by
+<p>Templates can be used in the form of a <b>list</b> as well. This has been 
+introduced with <b>6.5.0</b> The list consists of two parts which are either 
+a <b>constant</b> or a <b>property</b>. The constants
+are taking the part of "text" that you usually enter in string-based templates. 
+The properties stay variable, as they are a substitute for different values of a 
+certain type. This type of template is extremely useful for complicated cases,
+as it helps you to easily keep an overview over the template. Though, it has
+the disadvantage of needing more effort to create it.</p>
+<br>Config example:
+<br><blockquote><code>template(name="MyTemplate" type="list" option.json="off") {
+	<br>constant(value="Test: ")
+	<br>property(name="msg" outname="mymessage")
+	<br>constant(value=" --!!!-- ")
+	<br>property(name="timereported" dateFormat="rfc3339" caseConversion="lower")
+	<br>constant(value="\n")
+	<br>}
+</code></blockquote>
+<p>First, the general template option will be defined. The values of the template 
+itself get defined in the curly brackets. As it can be seen, we have constants
+and properties in exchange. Whereas constants will be filled with a value and probably
+some options, properties do direct to a property and the options that could be needed
+additional format definitions.</p>
+<p>We suggest to use separate lines for all constants and properties. This
+helps to keep a good overview over the different parts of the template.
+Though, writing it in a single line will work, it is much harder to debug
+if anything goes wrong with the template. </p>
+
+
+<p><b>Please note that templates can also be
 used to generate selector lines with dynamic file names.</b> For
 example, if you would like to split syslog messages from different
 hosts to different files (one per host), you can define the following
 template:</p>
+<blockquote><code>template (name="DynFile" type="string" string="/var/log/system-%HOSTNAME%.log")</code></blockquote>
+<p>Legacy example:</p>
 <blockquote><code>$template
 DynFile,"/var/log/system-%HOSTNAME%.log"</code></blockquote>
 <p>This template can then be used when defining an output
@@ -169,7 +210,7 @@ out, but this may happen.</li>
 is meant to be written to a log file. Do <b>not</b> use for production or remote
 forwarding.</li>
 </ul>
-<h3>String-based Template Samples</h3>
+<h3>Legacy String-based Template Samples</h3>
 <p>This section provides some sample of what the default formats would
 look as a text-based template. Hopefully, their description is self-explanatory.
 Note that each $Template statement is on a <b>single</b> line, but probably broken
-- 
cgit v1.2.3


From 72e1ccbc69d3b6047a98cfd45c00acab76c1d327 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 28 Aug 2012 09:59:15 +0200
Subject: preparing for 6.5.0 release

---
 ChangeLog       | 3 ++-
 doc/manual.html | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4ee5beb0..64dfd613 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,5 @@
 ---------------------------------------------------------------------------
-Version 6.5.0  [devel] 2012-0?-??
+Version 6.5.0  [devel] 2012-08-28
 - imrelp now supports non-cancel thread termination
   (but now requires at least librelp 1.0.1)
 - implemented freeCnf() module interface
@@ -18,6 +18,7 @@ Version 6.5.0  [devel] 2012-0?-??
   Thanks to Miloslav Trmač for the patch
 - $SystemLogParseTrusted config file option
   Thanks to Milan Bartos for the patch
+- added template config directive
 - added new uuid message property
   Thanks to Jérôme Renard for the idea and patches.
   Note: patches were released under ASL 2.0, see
diff --git a/doc/manual.html b/doc/manual.html
index de05d7ae..9a8644c9 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 6.4.0 (stable branch) of rsyslog.</b>
+<p><b>This documentation is for version 6.5.0 (devel branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
-- 
cgit v1.2.3


From 7348da7e02fae85ad24dd496f1cd8f6385cfcfa8 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 28 Aug 2012 19:07:58 +0200
Subject: bugfix: imtcp could abort on exit due to invalid free()

---
 ChangeLog             | 3 +++
 plugins/imtcp/imtcp.c | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 64dfd613..87beea2c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,7 @@
 ---------------------------------------------------------------------------
+Version 6.5.1  [devel] 2012-08-??
+- bugfix: imtcp could abort on exit due to invalid free()
+---------------------------------------------------------------------------
 Version 6.5.0  [devel] 2012-08-28
 - imrelp now supports non-cancel thread termination
   (but now requires at least librelp 1.0.1)
diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c
index a3365d44..f021307b 100644
--- a/plugins/imtcp/imtcp.c
+++ b/plugins/imtcp/imtcp.c
@@ -370,7 +370,6 @@ BEGINfreeCnf
 CODESTARTfreeCnf
 	for(inst = pModConf->root ; inst != NULL ; ) {
 		free(inst->pszBindPort);
-		free(inst->pBindRuleset);
 		free(inst->pszInputName);
 		del = inst;
 		inst = inst->next;
-- 
cgit v1.2.3


From 901270418eca404a1f05c2b5ace66954b8b690b1 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 29 Aug 2012 09:56:49 +0200
Subject: add OMSR_TPL_AS_JSON mode for output modules (interface only)

This patch does NOT contain any implementation, this will happen
next. Thanks to Miloslav Trmac, who suggested a similar mode, which
I based the JSON idea on.
---
 action.c              | 6 ++++++
 action.h              | 3 ++-
 doc/dev_oplugins.html | 5 +++++
 runtime/objomsr.c     | 3 ++-
 runtime/objomsr.h     | 7 ++++---
 5 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/action.c b/action.c
index 344bdc26..5d9211e2 100644
--- a/action.c
+++ b/action.c
@@ -823,6 +823,9 @@ static rsRetVal prepareDoActionParams(action_t *pAction, batch_obj_t *pElem)
 			case ACT_MSG_PASSING:
 				pElem->staticActParams[i] = (void*) pMsg;
 				break;
+			case ACT_JSON_PASSING:
+				// TODO: implement
+				break;
 			default:dbgprintf("software bug/error: unknown pAction->eParamPassing %d in prepareDoActionParams\n",
 					   (int) pAction->eParamPassing);
 				assert(0); /* software bug if this happens! */
@@ -874,6 +877,7 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch)
 				break;
 			case ACT_STRING_PASSING:
 			case ACT_MSG_PASSING:
+			case ACT_JSON_PASSING:
 				/* nothing to do in that case */
 				/* TODO ... and yet we do something ;) This is considered not
 				 * really needed, but I was not bold enough to remove that while
@@ -1933,6 +1937,8 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData,
 			pAction->eParamPassing = ACT_ARRAY_PASSING;
 		} else if(iTplOpts & OMSR_TPL_AS_MSG) {
 			pAction->eParamPassing = ACT_MSG_PASSING;
+		} else if(iTplOpts & OMSR_TPL_AS_JSON) {
+			pAction->eParamPassing = ACT_JSON_PASSING;
 		} else {
 			pAction->eParamPassing = ACT_STRING_PASSING;
 		}
diff --git a/action.h b/action.h
index 66ceaae5..0e704ddf 100644
--- a/action.h
+++ b/action.h
@@ -74,7 +74,8 @@ struct action_s {
 	int	f_repeatcount;	/* number of "repeated" msgs */
 	rsRetVal (*submitToActQ)(action_t *, batch_t *);/* function submit message to action queue */
 	rsRetVal (*qConstruct)(struct queue_s *pThis);
-	enum 	{ ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1, ACT_MSG_PASSING }
+	enum 	{ ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1, ACT_MSG_PASSING = 2,
+		  ACT_JSON_PASSING = 3}
 		eParamPassing;	/* mode of parameter passing to action */
 	int	iNumTpls;	/* number of array entries for template element below */
 	struct template **ppTpl;/* array of template to use - strings must be passed to doAction
diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html
index 63c186a3..b33b67f9 100644
--- a/doc/dev_oplugins.html
+++ b/doc/dev_oplugins.html
@@ -143,6 +143,11 @@ omstdout, you can see how a plugin may deal with the situation.
 array-passing capability not blindly be used.</b> In such cases, we can not guard the
 plugin from segfaulting and if the plugin (as currently always) is run within 
 rsyslog's process space, that results in a segfault for rsyslog. So do not do this.
+<p>Another possible mode is OMSR_TPL_AS_JSON, where instead of the template
+a json-c memory object tree is passed to the module. The module can extract data
+via json-c API calls. It MUST NOT modify the provided structure. This mode is 
+primarily aimed at plugins that need to process tree-like data, as found
+for example in MongoDB or ElasticSearch.
 <h3>Batching of Messages</h3>
 <p>Starting with rsyslog 4.3.x, batching of output messages is supported. Previously, only
 a single-message interface was supported.
diff --git a/runtime/objomsr.c b/runtime/objomsr.c
index 7241fa27..9cf3781b 100644
--- a/runtime/objomsr.c
+++ b/runtime/objomsr.c
@@ -149,7 +149,8 @@ OMSRgetSupportedTplOpts(unsigned long *pOpts)
 {
 	DEFiRet;
 	assert(pOpts != NULL);
-	*pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY | OMSR_TPL_AS_MSG;
+	*pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY | OMSR_TPL_AS_MSG
+		 | OMSR_TPL_AS_JSON;
 	RETiRet;
 }
 
diff --git a/runtime/objomsr.h b/runtime/objomsr.h
index 643e02c5..3baccaa3 100644
--- a/runtime/objomsr.h
+++ b/runtime/objomsr.h
@@ -25,12 +25,13 @@
 /* define flags for required template options */
 #define OMSR_NO_RQD_TPL_OPTS	0
 #define OMSR_RQD_TPL_OPT_SQL	1
-/* only one of OMSR_TPL_AS_ARRAY or _AS_MSG must be specified, if both are given
- * results are unpredictable.
+/* only one of OMSR_TPL_AS_ARRAY, _AS_MSG, or _AS_JSON  must be specified,
+ * if all are given results are unpredictable.
  */
 #define OMSR_TPL_AS_ARRAY	2	 /* introduced in 4.1.6, 2009-04-03 */
 #define OMSR_TPL_AS_MSG		4	 /* introduced in 5.3.4, 2009-11-02 */
-/* next option is 8, 16, 32, ... */
+#define OMSR_TPL_AS_JSON	8	 /* introduced in 6.5.1, 2012-09-02 */
+/* next option is 16, 32, 64, ... */
 
 struct omodStringRequest_s {	/* strings requested by output module for doAction() */
 	int iNumEntries;	/* number of array entries for data elements below */
-- 
cgit v1.2.3


From df492da1804b14e53a56a3dd6a3b8cc8fb1e2d78 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 29 Aug 2012 12:32:54 +0200
Subject: implement ACT_JSON_PASSING

NOTE: this is only compile-tested! For real testing, a module using
that mode is required. Will be done as next steps.
---
 action.c   | 23 +++++++++++------------
 template.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/action.c b/action.c
index 5d9211e2..8f8371f7 100644
--- a/action.c
+++ b/action.c
@@ -98,6 +98,7 @@
 #include <strings.h>
 #include <time.h>
 #include <errno.h>
+#include <json/json.h>
 
 #include "dirty.h"
 #include "template.h"
@@ -803,6 +804,7 @@ static rsRetVal prepareDoActionParams(action_t *pAction, batch_obj_t *pElem)
 {
 	int i;
 	msg_t *pMsg;
+	struct json_object *json;
 	DEFiRet;
 
 	ASSERT(pAction != NULL);
@@ -824,7 +826,8 @@ static rsRetVal prepareDoActionParams(action_t *pAction, batch_obj_t *pElem)
 				pElem->staticActParams[i] = (void*) pMsg;
 				break;
 			case ACT_JSON_PASSING:
-				// TODO: implement
+				CHKiRet(tplToJSON(pAction->ppTpl[i], pMsg, &json));
+				pElem->staticActParams[i] = (void*) json;
 				break;
 			default:dbgprintf("software bug/error: unknown pAction->eParamPassing %d in prepareDoActionParams\n",
 					   (int) pAction->eParamPassing);
@@ -877,7 +880,6 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch)
 				break;
 			case ACT_STRING_PASSING:
 			case ACT_MSG_PASSING:
-			case ACT_JSON_PASSING:
 				/* nothing to do in that case */
 				/* TODO ... and yet we do something ;) This is considered not
 				 * really needed, but I was not bold enough to remove that while
@@ -889,6 +891,13 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch)
 					((uchar**)pElem->staticActParams)[j] = NULL;
 				}
 				break;
+			case ACT_JSON_PASSING:
+				for(j = 0 ; j < pAction->iNumTpls ; ++j) {
+					json_object_put((struct json_object*)
+							pElem->staticActParams[j]);
+					pElem->staticActParams[j] = NULL;
+				}
+				break;
 			}
 		}
 	}
@@ -1098,16 +1107,6 @@ finalize_it:
 	RETiRet;
 }
 
-/* debug aid */
-static void displayBatchState(batch_t *pBatch)
-{
-	int i;
-	for(i = 0 ; i < pBatch->nElem ; ++i) {
-		dbgprintf("XXXXX: displayBatchState2 %p[%d]: %d\n", pBatch, i, pBatch->pElem[i].state);
-	}
-}
-
-
 /* submit a batch for actual action processing.
  * The first nElem elements are processed. This function calls itself
  * recursively if it needs to handle errors.
diff --git a/template.c b/template.c
index 768608f1..a3d2b8d6 100644
--- a/template.c
+++ b/template.c
@@ -34,6 +34,7 @@
 #include <string.h>
 #include <ctype.h>
 #include <assert.h>
+#include <json/json.h>
 #include "stringbuf.h"
 #include "syslogd-types.h"
 #include "template.h"
@@ -270,6 +271,50 @@ finalize_it:
 }
 
 
+/* This functions converts a template into a json object.
+ * For further general details, see the very similar funtion
+ * tpltoString().
+ * rgerhards, 2012-08-29
+ */
+rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson)
+{
+	struct templateEntry *pTpe;
+	char *cstr;
+	size_t propLen;
+	unsigned short bMustBeFreed;
+	uchar *pVal;
+	struct json_object *json, *jsonf;
+	DEFiRet;
+
+	assert(pTpl != NULL);
+	assert(pMsg != NULL);
+	assert(json != NULL);
+
+	json = json_object_new_object();
+	for(pTpe = pTpl->pEntryRoot ; pTpe != NULL ; pTpe = pTpe->pNext) {
+		if(pTpe->eEntryType == CONSTANT) {
+			if(pTpe->fieldName == NULL)
+				continue;
+			jsonf = json_object_new_string((char*) pTpe->data.constant.pConstant);
+		} else 	if(pTpe->eEntryType == FIELD) {
+			pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid,
+						   pTpe->data.field.propName,  &propLen, &bMustBeFreed);
+			jsonf = json_object_new_string_len((char*)pVal, propLen);
+			if(bMustBeFreed) { /* json-c makes its own private copy! */
+				free(pVal);
+			}
+		}
+		/* TODO: unify strings, handling currently quite inefficient! */
+		cstr = es_str2cstr(pTpe->fieldName, NULL);
+		json_object_object_add(json, cstr, jsonf);
+		free(cstr);
+	}
+
+	*pjson = (iRet == RS_RET_OK) ? json : NULL;
+	RETiRet;
+}
+
+
 /* Helper to doEscape. This is called if doEscape
  * runs out of memory allocating the escaped string.
  * Then we are in trouble. We can
-- 
cgit v1.2.3


From 1e09e2729e737e4512aa2205d72c537cce6d8cd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Fri, 24 Aug 2012 18:04:44 +0200
Subject: Split MongoDB output into BSON creation and MongoDB interaction
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This will allow us to add template processing more cleanly.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 plugins/ommongodb/ommongodb.c | 51 +++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c
index 00afcf68..a3848edb 100644
--- a/plugins/ommongodb/ommongodb.c
+++ b/plugins/ommongodb/ommongodb.c
@@ -205,13 +205,14 @@ i10pow(int exp)
 	}
 	return r;
 }
-/* write to mongodb in MSG passing mode, that is without a template.
+/* Return a BSON document when an user hasn't specified a template.
  * In this mode, we use the standard document format, which is somewhat
  * aligned to cee (as described in project lumberjack). Note that this is
  * a moving target, so we may run out of sync (and stay so to retain
  * backward compatibility, which we consider pretty important).
  */
-rsRetVal writeMongoDB_msg(msg_t *pMsg, instanceData *pData)
+static bson *
+getDefaultBSON(msg_t *pMsg)
 {
 	bson *doc = NULL;
 	uchar *procid; short unsigned procid_free; size_t procid_len;
@@ -222,12 +223,6 @@ rsRetVal writeMongoDB_msg(msg_t *pMsg, instanceData *pData)
 	int severity, facil;
 	gint64 ts_gen, ts_rcv; /* timestamps: generated, received */
 	int secfrac;
-	DEFiRet;
-
-	/* see if we are ready to proceed */
-	if(pData->conn == NULL) {
-		CHKiRet(initMongoDB(pData, 0));
-	}
 
 	procid = MsgGetProp(pMsg, NULL, PROP_PROGRAMNAME, NULL, &procid_len, &procid_free);
 	tag = MsgGetProp(pMsg, NULL, PROP_SYSLOGTAG, NULL, &tag_len, &tag_free);
@@ -279,22 +274,10 @@ dbgprintf("ommongodb: secfrac is %d, precision %d\n",  pMsg->tTIMESTAMP.secfrac,
 	if(sys_free) free(sys);
 	if(msg_free) free(msg);
 
-	if(doc == NULL) {
-		reportMongoError(pData);
-		dbgprintf("ommongodb: error creating BSON doc\n");
-		ABORT_FINALIZE(RS_RET_SUSPENDED);
-	}
+	if(doc == NULL)
+		return doc;
 	bson_finish(doc);
-	if(!mongo_sync_cmd_insert(pData->conn, (char*)pData->dbNcoll, doc, NULL)) {
-		reportMongoError(pData);
-		dbgprintf("ommongodb: insert error\n");
-		ABORT_FINALIZE(RS_RET_SUSPENDED);
-	}
-
-finalize_it:
-	if(doc != NULL)
-		bson_free(doc);
-	RETiRet;
+	return doc;
 }
 
 BEGINtryResume
@@ -305,10 +288,30 @@ CODESTARTtryResume
 ENDtryResume
 
 BEGINdoAction
+	bson *doc = NULL;
 CODESTARTdoAction
+	/* see if we are ready to proceed */
+	if(pData->conn == NULL) {
+		CHKiRet(initMongoDB(pData, 0));
+	}
+
 	if(pData->tplName == NULL) {
-		iRet = writeMongoDB_msg((msg_t*)ppString[0], pData);
+		doc = getDefaultBSON((msg_t*)ppString[0]);
+	}
+	if(doc == NULL) {
+		dbgprintf("ommongodb: error creating BSON doc\n");
+		/* FIXME: is this a correct return code? */
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	if(!mongo_sync_cmd_insert(pData->conn, (char*)pData->dbNcoll, doc, NULL)) {
+		reportMongoError(pData);
+		dbgprintf("ommongodb: insert error\n");
+		ABORT_FINALIZE(RS_RET_SUSPENDED);
 	}
+
+finalize_it:
+	if(doc != NULL)
+		bson_free(doc);
 ENDdoAction
 
 
-- 
cgit v1.2.3


From 9bd45fcb72e0e3a2abd89fc7fe0d068c0d68e1de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Fri, 24 Aug 2012 19:49:41 +0200
Subject: Add template support: parse text, convert to BSON
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Uses a json-c library (http://oss.metaparadigm.com/json-c/).

With this, a template
> $template MongoTemplate,"{%hostname:::jsonf:sys%, %$!all-json:2:$:%"
and action
> &    action(type="ommongodb" db="ceelog" collection="local" template="MongoTemplate")

will store a record created by
> logger -d -u p/dev/log -p mail.info -t mymailer '@cee: {"false": false, "null": null, "true": true, "object": {"a":"a", "b":"b"}, "array": [1, 2, 3], "number1": 3, "number2": 3.14, "string": "String"}'

as
> { "_id" : ObjectId("5037bbfc97dd811374ce5a00"), "sys" : "kulicka", "false" : "false", "null" : "-", "true" : "true", "object.a" : "a", "object.b" : "b", "array.*" : "3", "number1" : "3", "number2" : "3.140000", "string" : "String" }

which is not great, but fair enough given the current design of
libee (the information loss manifests by the time of template processing,
ommongodb can't fix it).

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 configure.ac                  |   7 +-
 plugins/ommongodb/Makefile.am |   4 +-
 plugins/ommongodb/ommongodb.c | 179 ++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 178 insertions(+), 12 deletions(-)

diff --git a/configure.ac b/configure.ac
index d0accfeb..0fe664a4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1279,14 +1279,13 @@ AC_ARG_ENABLE(ommongodb,
          esac],
         [enable_ommongodb=no]
 )
-#
-# you may want to do some library checks here - see snmp, mysql, pgsql modules
-# for samples
-#
 if test "x$enable_ommongodb" = "xyes"; then
 	PKG_CHECK_MODULES(LIBMONGO_CLIENT, libmongo-client >= 0.1.4)
 	AC_SUBST(LIBMONGO_CLIENT_CFLAGS)
 	AC_SUBST(LIBMONGO_CLIENT_LIBS)
+	PKG_CHECK_MODULES([JSON_C], [json])
+	AC_SUBST([JSON_CFLAGS])
+	AC_SUBST([JSON_LIBS])
 fi
 AM_CONDITIONAL(ENABLE_OMMONGODB, test x$enable_ommongodb = xyes)
 # end of mongodb code
diff --git a/plugins/ommongodb/Makefile.am b/plugins/ommongodb/Makefile.am
index 3a05c435..8b13c254 100644
--- a/plugins/ommongodb/Makefile.am
+++ b/plugins/ommongodb/Makefile.am
@@ -1,7 +1,7 @@
 pkglib_LTLIBRARIES = ommongodb.la
 ommongodb_la_SOURCES = ommongodb.c
-ommongodb_la_CPPFLAGS =  $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBMONGO_CLIENT_CFLAGS)
+ommongodb_la_CPPFLAGS =  $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBMONGO_CLIENT_CFLAGS) $(JSON_C_CFLAGS)
 ommongodb_la_LDFLAGS = -module -avoid-version
-ommongodb_la_LIBADD = $(LIBMONGO_CLIENT_LIBS)
+ommongodb_la_LIBADD = $(LIBMONGO_CLIENT_LIBS) $(JSON_C_LIBS)
 
 EXTRA_DIST = 
diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c
index a3848edb..9a7ca9f7 100644
--- a/plugins/ommongodb/ommongodb.c
+++ b/plugins/ommongodb/ommongodb.c
@@ -30,8 +30,12 @@
 #include <errno.h>
 #include <assert.h>
 #include <signal.h>
+#include <stdint.h>
 #include <time.h>
 #include <mongo.h>
+#include <json/json.h>
+/* For struct json_object_iter, should not be necessary in future versions */
+#include <json/json_object_private.h>
 
 #include "rsyslog.h"
 #include "conf.h"
@@ -42,6 +46,7 @@
 #include "datetime.h"
 #include "errmsg.h"
 #include "cfsysline.h"
+#include "unicode-helper.h"
 
 MODULE_TYPE_OUTPUT
 MODULE_TYPE_NOKEEP
@@ -54,6 +59,7 @@ DEFobjCurrIf(datetime)
 
 typedef struct _instanceData {
 	mongo_sync_connection *conn;
+	struct json_tokener *json_tokener; /* only if (tplName != NULL) */
 	uchar *server;
 	int port;
         uchar *db;
@@ -108,11 +114,14 @@ static void closeMongoDB(instanceData *pData)
 BEGINfreeInstance
 CODESTARTfreeInstance
 	closeMongoDB(pData);
+	if (pData->json_tokener != NULL)
+		json_tokener_free(pData->json_tokener);
 	free(pData->server);
 	free(pData->db);
 	free(pData->collection);
 	free(pData->uid);
 	free(pData->pwd);
+	free(pData->dbNcoll);
 	free(pData->tplName);
 ENDfreeInstance
 
@@ -280,6 +289,165 @@ dbgprintf("ommongodb: secfrac is %d, precision %d\n",  pMsg->tTIMESTAMP.secfrac,
 	return doc;
 }
 
+static bson *BSONFromJSONArray(struct json_object *json);
+static bson *BSONFromJSONObject(struct json_object *json);
+
+/* Append a BSON variant of json to doc using name.  Return TRUE on success */
+static gboolean
+BSONAppendJSONObject(bson *doc, const gchar *name, struct json_object *json)
+{
+	switch(json_object_get_type(json)) {
+	case json_type_boolean:
+		return bson_append_boolean(doc, name,
+					   json_object_get_boolean(json));
+	case json_type_double:
+		return bson_append_double(doc, name,
+					  json_object_get_double(json));
+	case json_type_int: {
+		int64_t i;
+
+		/* FIXME: the future version will have get_int64 */
+		i = json_object_get_int(json);
+		if (i >= INT32_MIN && i <= INT32_MAX)
+			return bson_append_int32(doc, name, i);
+		else
+			return bson_append_int64(doc, name, i);
+	}
+	case json_type_object: {
+		bson *sub;
+		gboolean ok;
+
+		sub = BSONFromJSONObject(json);
+		if (sub == NULL)
+			return FALSE;
+		ok = bson_append_document(doc, name, sub);
+		bson_free(sub);
+		return ok;
+	}
+	case json_type_array: {
+		bson *sub;
+		gboolean ok;
+
+		sub = BSONFromJSONArray(json);
+		if (sub == NULL)
+			return FALSE;
+		ok = bson_append_document(doc, name, sub);
+		bson_free(sub);
+		return ok;
+	}
+	case json_type_string:
+		return bson_append_string(doc, name,
+					  json_object_get_string(json), -1);
+
+	default:
+		return FALSE;
+	}
+}
+
+/* Return a BSON variant of json, which must be a json_type_array */
+static bson *
+BSONFromJSONArray(struct json_object *json)
+{
+	/* Way more than necessary */
+	bson *doc = NULL;
+	size_t i, array_len;
+
+	doc = bson_new();
+	if(doc == NULL)
+		goto error;
+
+	array_len = json_object_array_length(json);
+	for (i = 0; i < array_len; i++) {
+		char buf[sizeof(size_t) * CHAR_BIT + 1];
+
+		if ((size_t)snprintf(buf, sizeof(buf), "%zu", i) >= sizeof(buf))
+			goto error;
+		if (BSONAppendJSONObject(doc, buf,
+					 json_object_array_get_idx(json, i))
+		    == FALSE)
+			goto error;
+	}
+
+	if(bson_finish(doc) == FALSE)
+		goto error;
+
+	return doc;
+
+error:
+	if(doc != NULL)
+		bson_free(doc);
+	return NULL;
+}
+
+/* Return a BSON variant of json, which must be a json_type_object */
+static bson *
+BSONFromJSONObject(struct json_object *json)
+{
+	bson *doc = NULL;
+	struct json_object_iter it;
+
+	doc = bson_new();
+	if(doc == NULL)
+		goto error;
+
+	json_object_object_foreachC(json, it) {
+		if (BSONAppendJSONObject(doc, it.key, it.val) == FALSE)
+			goto error;
+	}
+
+	if(bson_finish(doc) == FALSE)
+		goto error;
+
+	return doc;
+
+error:
+	if(doc != NULL)
+		bson_free(doc);
+	return NULL;
+}
+
+/* Return a BSON document based on user's JSON string. */
+static bson *
+BSONFromJSONString(instanceData *pData, const char *json_string)
+{
+	size_t json_string_len;
+	struct json_object *json;
+	bson *doc = NULL;
+	const char *error_message;
+
+	json_tokener_reset(pData->json_tokener);
+
+	json_string_len = strlen(json_string);
+	json = json_tokener_parse_ex(pData->json_tokener,
+				     json_string, json_string_len);
+	error_message = NULL;
+	if(json == NULL) {
+		enum json_tokener_error err;
+
+		err = pData->json_tokener->err;
+		if(err != json_tokener_continue)
+			error_message = json_tokener_errors[err];
+		else
+			error_message = "Unterminated input";
+	} else if((size_t)pData->json_tokener->char_offset < json_string_len)
+		error_message = "Extra characters after JSON object";
+	else if(!json_object_is_type(json, json_type_object))
+		error_message = "JSON value is not an object";
+	if(error_message != NULL) {
+		/* FIXME: is dbgprintf() the correct way to report the error? */
+		dbgprintf("ommongodb: Error parsing JSON '%s': %s\n",
+			  json_string, error_message);
+		goto done;
+	}
+
+	doc = BSONFromJSONObject(json);
+
+done:
+	if(json != NULL)
+		json_object_put(json);
+	return doc;
+}
+
 BEGINtryResume
 CODESTARTtryResume
 	if(pData->conn == NULL) {
@@ -297,6 +465,8 @@ CODESTARTdoAction
 
 	if(pData->tplName == NULL) {
 		doc = getDefaultBSON((msg_t*)ppString[0]);
+	} else {
+		doc = BSONFromJSONString(pData, (const char *)ppString[0]);
 	}
 	if(doc == NULL) {
 		dbgprintf("ommongodb: error creating BSON doc\n");
@@ -366,12 +536,9 @@ CODESTARTnewActInst
 	if(pData->tplName == NULL) {
 		CHKiRet(OMSRsetEntry(*ppOMSR, 0, NULL, OMSR_TPL_AS_MSG));
 	} else {
-		errmsg.LogError(0, RS_RET_LEGA_ACT_NOT_SUPPORTED,
-			"ommongodb: templates are not supported in this version");
-		ABORT_FINALIZE(RS_RET_ERR);
-		CHKiRet(OMSRsetEntry(*ppOMSR, 0,
-			(uchar*) strdup((char*) pData->tplName),
-			OMSR_TPL_AS_ARRAY));
+		CHKiRet(OMSRsetEntry(*ppOMSR, 0, ustrdup(pData->tplName),
+				     OMSR_NO_RQD_TPL_OPTS));
+		CHKmalloc(pData->json_tokener = json_tokener_new());
 	}
 
 	if(pData->db == NULL)
-- 
cgit v1.2.3


From 081f366c576834554823c017ba950d0b48c687d7 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 29 Aug 2012 14:05:06 +0200
Subject: adapted Mirek's changes to new JSON passing mode

now the complete code (JSON passing & ommongodb) could be tested
and passed the module tests made during development
---
 configure.ac                  |  3 --
 plugins/ommongodb/Makefile.am |  4 +--
 plugins/ommongodb/ommongodb.c | 69 +++++++++++++++----------------------------
 runtime/rsyslog.h             |  1 +
 4 files changed, 27 insertions(+), 50 deletions(-)

diff --git a/configure.ac b/configure.ac
index 0fe664a4..1d7aec58 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1283,9 +1283,6 @@ if test "x$enable_ommongodb" = "xyes"; then
 	PKG_CHECK_MODULES(LIBMONGO_CLIENT, libmongo-client >= 0.1.4)
 	AC_SUBST(LIBMONGO_CLIENT_CFLAGS)
 	AC_SUBST(LIBMONGO_CLIENT_LIBS)
-	PKG_CHECK_MODULES([JSON_C], [json])
-	AC_SUBST([JSON_CFLAGS])
-	AC_SUBST([JSON_LIBS])
 fi
 AM_CONDITIONAL(ENABLE_OMMONGODB, test x$enable_ommongodb = xyes)
 # end of mongodb code
diff --git a/plugins/ommongodb/Makefile.am b/plugins/ommongodb/Makefile.am
index 8b13c254..3a05c435 100644
--- a/plugins/ommongodb/Makefile.am
+++ b/plugins/ommongodb/Makefile.am
@@ -1,7 +1,7 @@
 pkglib_LTLIBRARIES = ommongodb.la
 ommongodb_la_SOURCES = ommongodb.c
-ommongodb_la_CPPFLAGS =  $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBMONGO_CLIENT_CFLAGS) $(JSON_C_CFLAGS)
+ommongodb_la_CPPFLAGS =  $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBMONGO_CLIENT_CFLAGS)
 ommongodb_la_LDFLAGS = -module -avoid-version
-ommongodb_la_LIBADD = $(LIBMONGO_CLIENT_LIBS) $(JSON_C_LIBS)
+ommongodb_la_LIBADD = $(LIBMONGO_CLIENT_LIBS)
 
 EXTRA_DIST = 
diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c
index 9a7ca9f7..f2de33df 100644
--- a/plugins/ommongodb/ommongodb.c
+++ b/plugins/ommongodb/ommongodb.c
@@ -406,48 +406,6 @@ error:
 	return NULL;
 }
 
-/* Return a BSON document based on user's JSON string. */
-static bson *
-BSONFromJSONString(instanceData *pData, const char *json_string)
-{
-	size_t json_string_len;
-	struct json_object *json;
-	bson *doc = NULL;
-	const char *error_message;
-
-	json_tokener_reset(pData->json_tokener);
-
-	json_string_len = strlen(json_string);
-	json = json_tokener_parse_ex(pData->json_tokener,
-				     json_string, json_string_len);
-	error_message = NULL;
-	if(json == NULL) {
-		enum json_tokener_error err;
-
-		err = pData->json_tokener->err;
-		if(err != json_tokener_continue)
-			error_message = json_tokener_errors[err];
-		else
-			error_message = "Unterminated input";
-	} else if((size_t)pData->json_tokener->char_offset < json_string_len)
-		error_message = "Extra characters after JSON object";
-	else if(!json_object_is_type(json, json_type_object))
-		error_message = "JSON value is not an object";
-	if(error_message != NULL) {
-		/* FIXME: is dbgprintf() the correct way to report the error? */
-		dbgprintf("ommongodb: Error parsing JSON '%s': %s\n",
-			  json_string, error_message);
-		goto done;
-	}
-
-	doc = BSONFromJSONObject(json);
-
-done:
-	if(json != NULL)
-		json_object_put(json);
-	return doc;
-}
-
 BEGINtryResume
 CODESTARTtryResume
 	if(pData->conn == NULL) {
@@ -466,7 +424,7 @@ CODESTARTdoAction
 	if(pData->tplName == NULL) {
 		doc = getDefaultBSON((msg_t*)ppString[0]);
 	} else {
-		doc = BSONFromJSONString(pData, (const char *)ppString[0]);
+		doc = BSONFromJSONObject((struct json_object *)ppString[0]);
 	}
 	if(doc == NULL) {
 		dbgprintf("ommongodb: error creating BSON doc\n");
@@ -537,7 +495,7 @@ CODESTARTnewActInst
 		CHKiRet(OMSRsetEntry(*ppOMSR, 0, NULL, OMSR_TPL_AS_MSG));
 	} else {
 		CHKiRet(OMSRsetEntry(*ppOMSR, 0, ustrdup(pData->tplName),
-				     OMSR_NO_RQD_TPL_OPTS));
+				     OMSR_TPL_AS_JSON));
 		CHKmalloc(pData->json_tokener = json_tokener_new());
 	}
 
@@ -590,6 +548,10 @@ CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES
 ENDqueryEtryPt
 
 BEGINmodInit()
+	rsRetVal localRet;
+	rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts);
+	unsigned long opts;
+	int bJSONPassingSupported;
 CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
@@ -597,5 +559,22 @@ CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(datetime, CORE_COMPONENT));
 	INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING);
 	DBGPRINTF("ommongodb: module compiled with rsyslog version %s.\n", VERSION);
-	//DBGPRINTF("ommongodb: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not ");
+
+	/* check if the rsyslog core supports parameter passing code */
+	bJSONPassingSupported = 0;
+	localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts",
+				    &pomsrGetSupportedTplOpts);
+	if(localRet == RS_RET_OK) {
+		/* found entry point, so let's see if core supports msg passing */
+		CHKiRet((*pomsrGetSupportedTplOpts)(&opts));
+		if(opts & OMSR_TPL_AS_JSON)
+			bJSONPassingSupported = 1;
+	} else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) {
+		ABORT_FINALIZE(localRet); /* Something else went wrong, not acceptable */
+	}
+	if(!bJSONPassingSupported) {
+		DBGPRINTF("ommongodb: JSON-passing is not supported by rsyslog core, "
+			  "can not continue.\n");
+		ABORT_FINALIZE(RS_RET_NO_JSON_PASSING);
+	}
 ENDmodInit
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 57e8a05c..928c3ab9 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -380,6 +380,7 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_DUP_PARAM = -2220, /**< config parameter is given more than once */
 	RS_RET_MODULE_ALREADY_IN_CONF = -2221, /**< module already in current configuration */
 	RS_RET_PARAM_NOT_PERMITTED = -2222, /**< legacy parameter no longer permitted (usally already set by v2) */
+	RS_RET_NO_JSON_PASSING = -2223, /**< rsyslog core does not support JSON-passing plugin API */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */
-- 
cgit v1.2.3


From a5e51ced933cb73bd3755bffb81aceceb3f0f953 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 29 Aug 2012 14:10:18 +0200
Subject: milestone: ommongodb supports templates & JSON passing API in place

---
 ChangeLog                |  2 ++
 plugins/ommongodb/README | 23 ++++++++---------------
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 87beea2c..f4964760 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 6.5.1  [devel] 2012-08-??
+- added pure JSON output plugin parameter passing mode
+- ommongodb now supports templates
 - bugfix: imtcp could abort on exit due to invalid free()
 ---------------------------------------------------------------------------
 Version 6.5.0  [devel] 2012-08-28
diff --git a/plugins/ommongodb/README b/plugins/ommongodb/README
index 7581131a..ad4a8ea2 100644
--- a/plugins/ommongodb/README
+++ b/plugins/ommongodb/README
@@ -6,20 +6,13 @@ configuration:
 
 in your /etc/rsyslog.conf, together with other modules:
 $ModLoad ommongodb # provides mongodb support
+*.*     action(type="ommongodb" db="..." collection="..." template="...")
 
-then in your /etc/rsyslog.d (check your distribution way to organize the configuration..) you create a file 10-mongodb.conf with the following content:
+Note: if no template is specified, a default schema will be used. That schema
+contains proper data types. However, if a template is specified, only strings
+are supported. This is a restriction of the rsyslog v6 core engine. This 
+changed in v7.
 
-*.*     action(type="ommongodb" db="..." collection="...")
-
-Note: currently templates are not supported. Ommongodb will pick a default
-schema and use the message object content for that (templateless).
-
-
-TODO
-we must ensure that the collection is a capped collection
-refactor my code :-)
-
-email Victor Pereira <victor.pereira@bigrails.com>
-twitter twitter.com/vpereira
-
-part of this doc by Rainer Gerhards <rgerhards@adiscon.com>
+If templates are used, it is suggested to use list-based templates. Constants
+can ONLY be inserted with list-based templates, as only these provide the
+capability to specify a field name (outname parameter).
-- 
cgit v1.2.3