diff options
30 files changed, 1622 insertions, 57 deletions
@@ -7,6 +7,8 @@ Version 7.3.2 [devel] 2012-10-?? * support for Linux-Type ratelimiting added - permited action-like statements (stop, call, ...) in action lists - bugfix: segfault on startup when modules using MSG_PASSING mode are used +- omelasticsearch: support for writing data errors to local file added +- omelasticsearch: fix check for bulk processing status response --------------------------------------------------------------------------- Version 7.3.1 [devel] 2012-10-19 - optimized template processing performance, especially for $NOW family @@ -34,7 +36,13 @@ Version 7.3.0 [devel] 2012-10-09 This was achieved by somewhat reducing the robustness of the zip archive. This is controlled by the new action parameter "VeryReliableZip". ---------------------------------------------------------------------------- -Version 7.2.1 [v7-stable] 2012-10-?? +Version 7.2.2 [v7-stable] 2012-10-30 +- bugfix: omfwd did not properly support "template" parameter +---------------------------------------------------------------------------- +Version 7.2.1 [v7-stable] 2012-10-29 +- bugfix: ruleset()-object did only support a single statement +- added -D rsyslogd option to enable config parser debug mode +- improved syntax error messages by outputting the error token - the rsyslog core now suspeneds actions after 10 failures in a row This was former the case after 1,000 failures and could cause rsyslog to be spammed/ressources misused. See the v6 compatibility doc for more diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index 3e4bceeb..0c189100 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -188,7 +188,7 @@ templates are a great way to specify textual content, especially if no complex manipulation to properties is necessary. Full details on how to specify template text can be found below. <br>Config example: -<br><blockquote><code>template(name="tpl3" type="list" string="%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n") +<br><blockquote><code>template(name="tpl3" type="string" string="%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n") </code></blockquote> <h3>plugin</h3> In this case, the template is generated by a plugin (which diff --git a/doc/v6compatibility.html b/doc/v6compatibility.html index edb6dedf..7ce8c001 100644 --- a/doc/v6compatibility.html +++ b/doc/v6compatibility.html @@ -181,7 +181,7 @@ As you see, here you may include spaces between user names. so it is a wise decision to change config files at least to the legacy format (with ":omusrmsg:" in front of the name). -<h2>Escape Sequences in Script-Based Filters<h2> +<h2>Escape Sequences in Script-Based Filters</h2> <p>In v5, escape sequences were very simplistic. Inside a string, "\x" meant "x" with x being any character. This has been changed so that the usual set of escapes is supported, must importantly "\n", "\t", "\xhh" (with hh being hex digits) @@ -189,7 +189,6 @@ and "\ooo" with (o being octal digits). So if one of these sequences was used previously, results are obviously different. However, that should not create any real problems, because it is hard to envision why someone should have done that (why write "\n" when you can also write "n"?). ---------------------------------------------------------------------------- <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> diff --git a/grammar/grammar.y b/grammar/grammar.y index 42da0efc..c5bad689 100644 --- a/grammar/grammar.y +++ b/grammar/grammar.y @@ -131,7 +131,7 @@ obj: BEGINOBJ nvlst ENDOBJ { $$ = cnfobjNew($1, $2); } { $$ = cnfobjNew(CNFOBJ_TPL, $2); $$->subobjs = $5; } - | BEGIN_RULESET nvlst ENDOBJ '{' stmt '}' + | BEGIN_RULESET nvlst ENDOBJ '{' script '}' { $$ = cnfobjNew(CNFOBJ_RULESET, $2); $$->script = $5; } diff --git a/plugins/omelasticsearch/Makefile.am b/plugins/omelasticsearch/Makefile.am index a574c72f..059bdf8a 100644 --- a/plugins/omelasticsearch/Makefile.am +++ b/plugins/omelasticsearch/Makefile.am @@ -1,6 +1,7 @@ pkglib_LTLIBRARIES = omelasticsearch.la -omelasticsearch_la_SOURCES = omelasticsearch.c +# TODO: replace cJSON +omelasticsearch_la_SOURCES = omelasticsearch.c cJSON/cjson.c cJSON/cjson.h omelasticsearch_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) omelasticsearch_la_LDFLAGS = -module -avoid-version omelasticsearch_la_LIBADD = $(CURL_LIBS) diff --git a/plugins/omelasticsearch/README b/plugins/omelasticsearch/README new file mode 100644 index 00000000..9021bc0e --- /dev/null +++ b/plugins/omelasticsearch/README @@ -0,0 +1,17 @@ +How to produce an error: +======================== +It's quite easy to get 400, if you put a wrong mapping to your +index. That would be easy to reproduce in "normal" omelasticsearch usage +conditions, by only altering the ES configuration: + +1. Make your index first. Let's call it "testindex": +$ curl -XPUT localhost:9200/testindex/ + +2. Put your mapping for a search type called "mytype", where you specify +that date property should be an integer: +$ curl -XPUT localhost:9200/testindex/mytype/_mapping -d '{"mytype":{"properties": {"timegenerated":{"type":"integer"}}}}' + +3. Now try to insert something where date is not an integer: +$ curl -XPOST localhost:9200/testindex/mytype/ -d '{"timegenerated":"bla"}' +{"error":"MapperParsingException[Failed to parse [date]]; nested: NumberFormatException[For input string: \"bla\"]; ","status":400} + diff --git a/plugins/omelasticsearch/cJSON/README b/plugins/omelasticsearch/cJSON/README new file mode 100644 index 00000000..7531c049 --- /dev/null +++ b/plugins/omelasticsearch/cJSON/README @@ -0,0 +1,247 @@ +/* + Copyright (c) 2009 Dave Gamble + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +Welcome to cJSON. + +cJSON aims to be the dumbest possible parser that you can get your job done with. +It's a single file of C, and a single header file. + +JSON is described best here: http://www.json.org/ +It's like XML, but fat-free. You use it to move data around, store things, or just +generally represent your program's state. + + +First up, how do I build? +Add cJSON.c to your project, and put cJSON.h somewhere in the header search path. +For example, to build the test app: + +gcc cJSON.c test.c -o test -lm +./test + + +As a library, cJSON exists to take away as much legwork as it can, but not get in your way. +As a point of pragmatism (i.e. ignoring the truth), I'm going to say that you can use it +in one of two modes: Auto and Manual. Let's have a quick run-through. + + +I lifted some JSON from this page: http://www.json.org/fatfree.html +That page inspired me to write cJSON, which is a parser that tries to share the same +philosophy as JSON itself. Simple, dumb, out of the way. + +Some JSON: +{ + "name": "Jack (\"Bee\") Nimble", + "format": { + "type": "rect", + "width": 1920, + "height": 1080, + "interlace": false, + "frame rate": 24 + } +} + +Assume that you got this from a file, a webserver, or magic JSON elves, whatever, +you have a char * to it. Everything is a cJSON struct. +Get it parsed: + cJSON *root = cJSON_Parse(my_json_string); + +This is an object. We're in C. We don't have objects. But we do have structs. +What's the framerate? + + cJSON *format = cJSON_GetObjectItem(root,"format"); + int framerate = cJSON_GetObjectItem(format,"frame rate")->valueint; + + +Want to change the framerate? + cJSON_GetObjectItem(format,"frame rate")->valueint=25; + +Back to disk? + char *rendered=cJSON_Print(root); + +Finished? Delete the root (this takes care of everything else). + cJSON_Delete(root); + +That's AUTO mode. If you're going to use Auto mode, you really ought to check pointers +before you dereference them. If you want to see how you'd build this struct in code? + cJSON *root,*fmt; + root=cJSON_CreateObject(); + cJSON_AddItemToObject(root, "name", cJSON_CreateString("Jack (\"Bee\") Nimble")); + cJSON_AddItemToObject(root, "format", fmt=cJSON_CreateObject()); + cJSON_AddStringToObject(fmt,"type", "rect"); + cJSON_AddNumberToObject(fmt,"width", 1920); + cJSON_AddNumberToObject(fmt,"height", 1080); + cJSON_AddFalseToObject (fmt,"interlace"); + cJSON_AddNumberToObject(fmt,"frame rate", 24); + +Hopefully we can agree that's not a lot of code? There's no overhead, no unnecessary setup. +Look at test.c for a bunch of nice examples, mostly all ripped off the json.org site, and +a few from elsewhere. + +What about manual mode? First up you need some detail. +Let's cover how the cJSON objects represent the JSON data. +cJSON doesn't distinguish arrays from objects in handling; just type. +Each cJSON has, potentially, a child, siblings, value, a name. + +The root object has: Object Type and a Child +The Child has name "name", with value "Jack ("Bee") Nimble", and a sibling: +Sibling has type Object, name "format", and a child. +That child has type String, name "type", value "rect", and a sibling: +Sibling has type Number, name "width", value 1920, and a sibling: +Sibling has type Number, name "height", value 1080, and a sibling: +Sibling hs type False, name "interlace", and a sibling: +Sibling has type Number, name "frame rate", value 24 + +Here's the structure: +typedef struct cJSON { + struct cJSON *next,*prev; + struct cJSON *child; + + int type; + + char *valuestring; + int valueint; + double valuedouble; + + char *string; +} cJSON; + +By default all values are 0 unless set by virtue of being meaningful. + +next/prev is a doubly linked list of siblings. next takes you to your sibling, +prev takes you back from your sibling to you. +Only objects and arrays have a "child", and it's the head of the doubly linked list. +A "child" entry will have prev==0, but next potentially points on. The last sibling has next=0. +The type expresses Null/True/False/Number/String/Array/Object, all of which are #defined in +cJSON.h + +A Number has valueint and valuedouble. If you're expecting an int, read valueint, if not read +valuedouble. + +Any entry which is in the linked list which is the child of an object will have a "string" +which is the "name" of the entry. When I said "name" in the above example, that's "string". +"string" is the JSON name for the 'variable name' if you will. + +Now you can trivially walk the lists, recursively, and parse as you please. +You can invoke cJSON_Parse to get cJSON to parse for you, and then you can take +the root object, and traverse the structure (which is, formally, an N-tree), +and tokenise as you please. If you wanted to build a callback style parser, this is how +you'd do it (just an example, since these things are very specific): + +void parse_and_callback(cJSON *item,const char *prefix) +{ + while (item) + { + char *newprefix=malloc(strlen(prefix)+strlen(item->name)+2); + sprintf(newprefix,"%s/%s",prefix,item->name); + int dorecurse=callback(newprefix, item->type, item); + if (item->child && dorecurse) parse_and_callback(item->child,newprefix); + item=item->next; + free(newprefix); + } +} + +The prefix process will build you a separated list, to simplify your callback handling. +The 'dorecurse' flag would let the callback decide to handle sub-arrays on it's own, or +let you invoke it per-item. For the item above, your callback might look like this: + +int callback(const char *name,int type,cJSON *item) +{ + if (!strcmp(name,"name")) { /* populate name */ } + else if (!strcmp(name,"format/type") { /* handle "rect" */ } + else if (!strcmp(name,"format/width") { /* 800 */ } + else if (!strcmp(name,"format/height") { /* 600 */ } + else if (!strcmp(name,"format/interlace") { /* false */ } + else if (!strcmp(name,"format/frame rate") { /* 24 */ } + return 1; +} + +Alternatively, you might like to parse iteratively. +You'd use: + +void parse_object(cJSON *item) +{ + int i; for (i=0;i<cJSON_GetArraySize(item);i++) + { + cJSON *subitem=cJSON_GetArrayItem(item,i); + // handle subitem. + } +} + +Or, for PROPER manual mode: + +void parse_object(cJSON *item) +{ + cJSON *subitem=item->child; + while (subitem) + { + // handle subitem + if (subitem->child) parse_object(subitem->child); + + subitem=subitem->next; + } +} + +Of course, this should look familiar, since this is just a stripped-down version +of the callback-parser. + +This should cover most uses you'll find for parsing. The rest should be possible +to infer.. and if in doubt, read the source! There's not a lot of it! ;) + + +In terms of constructing JSON data, the example code above is the right way to do it. +You can, of course, hand your sub-objects to other functions to populate. +Also, if you find a use for it, you can manually build the objects. +For instance, suppose you wanted to build an array of objects? + +cJSON *objects[24]; + +cJSON *Create_array_of_anything(cJSON **items,int num) +{ + int i;cJSON *prev, *root=cJSON_CreateArray(); + for (i=0;i<24;i++) + { + if (!i) root->child=objects[i]; + else prev->next=objects[i], objects[i]->prev=prev; + prev=objects[i]; + } + return root; +} + +and simply: Create_array_of_anything(objects,24); + +cJSON doesn't make any assumptions about what order you create things in. +You can attach the objects, as above, and later add children to each +of those objects. + +As soon as you call cJSON_Print, it renders the structure to text. + + + +The test.c code shows how to handle a bunch of typical cases. If you uncomment +the code, it'll load, parse and print a bunch of test files, also from json.org, +which are more complex than I'd care to try and stash into a const char array[]. + + +Enjoy cJSON! + + +- Dave Gamble, Aug 2009 diff --git a/plugins/omelasticsearch/cJSON/cjson.c b/plugins/omelasticsearch/cJSON/cjson.c new file mode 100644 index 00000000..99a831e9 --- /dev/null +++ b/plugins/omelasticsearch/cJSON/cjson.c @@ -0,0 +1,514 @@ +/* + Copyright (c) 2009 Dave Gamble + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +/* cJSON */ +/* JSON parser in C. */ + +#include <string.h> +#include <stdio.h> +#include <math.h> +#include <stdlib.h> +#include <float.h> +#include <limits.h> +#include <ctype.h> +#include "cjson.h" + +static const char *ep; + +const char *cJSON_GetErrorPtr() {return ep;} + +static int cJSON_strcasecmp(const char *s1,const char *s2) +{ + if (!s1) return (s1==s2)?0:1;if (!s2) return 1; + for(; tolower(*s1) == tolower(*s2); ++s1, ++s2) if(*s1 == 0) return 0; + return tolower(*(const unsigned char *)s1) - tolower(*(const unsigned char *)s2); +} + +static void *(*cJSON_malloc)(size_t sz) = malloc; +static void (*cJSON_free)(void *ptr) = free; + +static char* cJSON_strdup(const char* str) +{ + size_t len; + char* copy; + + len = strlen(str) + 1; + if (!(copy = (char*)cJSON_malloc(len))) return 0; + memcpy(copy,str,len); + return copy; +} + +void cJSON_InitHooks(cJSON_Hooks* hooks) +{ + if (!hooks) { /* Reset hooks */ + cJSON_malloc = malloc; + cJSON_free = free; + return; + } + + cJSON_malloc = (hooks->malloc_fn)?hooks->malloc_fn:malloc; + cJSON_free = (hooks->free_fn)?hooks->free_fn:free; +} + +/* Internal constructor. */ +static cJSON *cJSON_New_Item() +{ + cJSON* node = (cJSON*)cJSON_malloc(sizeof(cJSON)); + if (node) memset(node,0,sizeof(cJSON)); + return node; +} + +/* Delete a cJSON structure. */ +void cJSON_Delete(cJSON *c) +{ + cJSON *next; + while (c) + { + next=c->next; + if (!(c->type&cJSON_IsReference) && c->child) cJSON_Delete(c->child); + if (!(c->type&cJSON_IsReference) && c->valuestring) cJSON_free(c->valuestring); + if (c->string) cJSON_free(c->string); + cJSON_free(c); + c=next; + } +} + +/* Parse the input text to generate a number, and populate the result into item. */ +static const char *parse_number(cJSON *item,const char *num) +{ + double n=0,sign=1,scale=0;int subscale=0,signsubscale=1; + + /* Could use sscanf for this? */ + if (*num=='-') sign=-1,num++; /* Has sign? */ + if (*num=='0') num++; /* is zero */ + if (*num>='1' && *num<='9') do n=(n*10.0)+(*num++ -'0'); while (*num>='0' && *num<='9'); /* Number? */ + if (*num=='.' && num[1]>='0' && num[1]<='9') {num++; do n=(n*10.0)+(*num++ -'0'),scale--; while (*num>='0' && *num<='9');} /* Fractional part? */ + if (*num=='e' || *num=='E') /* Exponent? */ + { num++;if (*num=='+') num++; else if (*num=='-') signsubscale=-1,num++; /* With sign? */ + while (*num>='0' && *num<='9') subscale=(subscale*10)+(*num++ - '0'); /* Number? */ + } + + n=sign*n*pow(10.0,(scale+subscale*signsubscale)); /* number = +/- number.fraction * 10^+/- exponent */ + + item->valuedouble=n; + item->valueint=(int)n; + item->type=cJSON_Number; + return num; +} + +/* Render the number nicely from the given item into a string. */ +char *cJSON_print_number(cJSON *item) +{ + char *str; + double d=item->valuedouble; + if (fabs(((double)item->valueint)-d)<=DBL_EPSILON && d<=INT_MAX && d>=INT_MIN) + { + str=(char*)cJSON_malloc(21); /* 2^64+1 can be represented in 21 chars. */ + if (str) sprintf(str,"%d",item->valueint); + } + else + { + str=(char*)cJSON_malloc(64); /* This is a nice tradeoff. */ + if (str) + { + if (fabs(floor(d)-d)<=DBL_EPSILON) sprintf(str,"%.0f",d); + else if (fabs(d)<1.0e-6 || fabs(d)>1.0e9) sprintf(str,"%e",d); + else sprintf(str,"%f",d); + } + } + return str; +} + +/* Parse the input text into an unescaped cstring, and populate item. */ +static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; +static const char *parse_string(cJSON *item,const char *str) +{ + const char *ptr=str+1;char *ptr2;char *out;int len=0;unsigned uc,uc2; + if (*str!='\"') {ep=str;return 0;} /* not a string! */ + + while (*ptr!='\"' && *ptr && ++len) if (*ptr++ == '\\') ptr++; /* Skip escaped quotes. */ + + out=(char*)cJSON_malloc(len+1); /* This is how long we need for the string, roughly. */ + if (!out) return 0; + + ptr=str+1;ptr2=out; + while (*ptr!='\"' && *ptr) + { + if (*ptr!='\\') *ptr2++=*ptr++; + else + { + ptr++; + switch (*ptr) + { + case 'b': *ptr2++='\b'; break; + case 'f': *ptr2++='\f'; break; + case 'n': *ptr2++='\n'; break; + case 'r': *ptr2++='\r'; break; + case 't': *ptr2++='\t'; break; + case 'u': /* transcode utf16 to utf8. */ + sscanf(ptr+1,"%4x",&uc);ptr+=4; /* get the unicode char. */ + + if ((uc>=0xDC00 && uc<=0xDFFF) || uc==0) break; // check for invalid. + + if (uc>=0xD800 && uc<=0xDBFF) // UTF16 surrogate pairs. + { + if (ptr[1]!='\\' || ptr[2]!='u') break; // missing second-half of surrogate. + sscanf(ptr+3,"%4x",&uc2);ptr+=6; + if (uc2<0xDC00 || uc2>0xDFFF) break; // invalid second-half of surrogate. + uc=0x10000 | ((uc&0x3FF)<<10) | (uc2&0x3FF); + } + + len=4;if (uc<0x80) len=1;else if (uc<0x800) len=2;else if (uc<0x10000) len=3; ptr2+=len; + + switch (len) { + case 4: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 3: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 2: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 1: *--ptr2 =(uc | firstByteMark[len]); + } + ptr2+=len; + break; + default: *ptr2++=*ptr; break; + } + ptr++; + } + } + *ptr2=0; + if (*ptr=='\"') ptr++; + item->valuestring=out; + item->type=cJSON_String; + return ptr; +} + +/* Render the cstring provided to an escaped version that can be printed. */ +static char *print_string_ptr(const char *str) +{ + const char *ptr;char *ptr2,*out;int len=0;unsigned char token; + + if (!str) return cJSON_strdup(""); + ptr=str;while ((token=*ptr) && ++len) {if (strchr("\"\\\b\f\n\r\t",token)) len++; else if (token<32) len+=5;ptr++;} + + out=(char*)cJSON_malloc(len+3); + if (!out) return 0; + + ptr2=out;ptr=str; + *ptr2++='\"'; + while (*ptr) + { + if ((unsigned char)*ptr>31 && *ptr!='\"' && *ptr!='\\') *ptr2++=*ptr++; + else + { + *ptr2++='\\'; + switch (token=*ptr++) + { + case '\\': *ptr2++='\\'; break; + case '\"': *ptr2++='\"'; break; + case '\b': *ptr2++='b'; break; + case '\f': *ptr2++='f'; break; + case '\n': *ptr2++='n'; break; + case '\r': *ptr2++='r'; break; + case '\t': *ptr2++='t'; break; + default: sprintf(ptr2,"u%04x",token);ptr2+=5; break; /* escape and print */ + } + } + } + *ptr2++='\"';*ptr2++=0; + return out; +} +/* Invote print_string_ptr (which is useful) on an item. */ +static char *print_string(cJSON *item) {return print_string_ptr(item->valuestring);} + +/* Predeclare these prototypes. */ +static const char *parse_value(cJSON *item,const char *value); +static char *print_value(cJSON *item,int depth,int fmt); +static const char *parse_array(cJSON *item,const char *value); +static char *print_array(cJSON *item,int depth,int fmt); +static const char *parse_object(cJSON *item,const char *value); +static char *print_object(cJSON *item,int depth,int fmt); + +/* Utility to jump whitespace and cr/lf */ +static const char *skip(const char *in) {while (in && *in && (unsigned char)*in<=32) in++; return in;} + +/* Parse an object - create a new root, and populate. */ +cJSON *cJSON_Parse(const char *value) +{ + cJSON *c=cJSON_New_Item(); + ep=0; + if (!c) return 0; /* memory fail */ + + if (!parse_value(c,skip(value))) {cJSON_Delete(c);return 0;} + return c; +} + +/* Render a cJSON item/entity/structure to text. */ +char *cJSON_Print(cJSON *item) {return print_value(item,0,1);} +char *cJSON_PrintUnformatted(cJSON *item) {return print_value(item,0,0);} + +/* Parser core - when encountering text, process appropriately. */ +static const char *parse_value(cJSON *item,const char *value) +{ + if (!value) return 0; /* Fail on null. */ + if (!strncmp(value,"null",4)) { item->type=cJSON_NULL; return value+4; } + if (!strncmp(value,"false",5)) { item->type=cJSON_False; return value+5; } + if (!strncmp(value,"true",4)) { item->type=cJSON_True; item->valueint=1; return value+4; } + if (*value=='\"') { return parse_string(item,value); } + if (*value=='-' || (*value>='0' && *value<='9')) { return parse_number(item,value); } + if (*value=='[') { return parse_array(item,value); } + if (*value=='{') { return parse_object(item,value); } + + ep=value;return 0; /* failure. */ +} + +/* Render a value to text. */ +static char *print_value(cJSON *item,int depth,int fmt) +{ + char *out=0; + if (!item) return 0; + switch ((item->type)&255) + { + case cJSON_NULL: out=cJSON_strdup("null"); break; + case cJSON_False: out=cJSON_strdup("false");break; + case cJSON_True: out=cJSON_strdup("true"); break; + case cJSON_Number: out=cJSON_print_number(item);break; + case cJSON_String: out=print_string(item);break; + case cJSON_Array: out=print_array(item,depth,fmt);break; + case cJSON_Object: out=print_object(item,depth,fmt);break; + } + return out; +} + +/* Build an array from input text. */ +static const char *parse_array(cJSON *item,const char *value) +{ + cJSON *child; + if (*value!='[') {ep=value;return 0;} /* not an array! */ + + item->type=cJSON_Array; + value=skip(value+1); + if (*value==']') return value+1; /* empty array. */ + + item->child=child=cJSON_New_Item(); + if (!item->child) return 0; /* memory fail */ + value=skip(parse_value(child,skip(value))); /* skip any spacing, get the value. */ + if (!value) return 0; + + while (*value==',') + { + cJSON *new_item; + if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */ + child->next=new_item;new_item->prev=child;child=new_item; + value=skip(parse_value(child,skip(value+1))); + if (!value) return 0; /* memory fail */ + } + + if (*value==']') return value+1; /* end of array */ + ep=value;return 0; /* malformed. */ +} + +/* Render an array to text */ +static char *print_array(cJSON *item,int depth,int fmt) +{ + char **entries; + char *out=0,*ptr,*ret;int len=5; + cJSON *child=item->child; + int numentries=0,i=0,fail=0; + + /* How many entries in the array? */ + while (child) numentries++,child=child->next; + /* Allocate an array to hold the values for each */ + entries=(char**)cJSON_malloc(numentries*sizeof(char*)); + if (!entries) return 0; + memset(entries,0,numentries*sizeof(char*)); + /* Retrieve all the results: */ + child=item->child; + while (child && !fail) + { + ret=print_value(child,depth+1,fmt); + entries[i++]=ret; + if (ret) len+=strlen(ret)+2+(fmt?1:0); else fail=1; + child=child->next; + } + + /* If we didn't fail, try to malloc the output string */ + if (!fail) out=(char*)cJSON_malloc(len); + /* If that fails, we fail. */ + if (!out) fail=1; + + /* Handle failure. */ + if (fail) + { + for (i=0;i<numentries;i++) if (entries[i]) cJSON_free(entries[i]); + cJSON_free(entries); + return 0; + } + + /* Compose the output array. */ + *out='['; + ptr=out+1;*ptr=0; + for (i=0;i<numentries;i++) + { + strcpy(ptr,entries[i]);ptr+=strlen(entries[i]); + if (i!=numentries-1) {*ptr++=',';if(fmt)*ptr++=' ';*ptr=0;} + cJSON_free(entries[i]); + } + cJSON_free(entries); + *ptr++=']';*ptr++=0; + return out; +} + +/* Build an object from the text. */ +static const char *parse_object(cJSON *item,const char *value) +{ + cJSON *child; + if (*value!='{') {ep=value;return 0;} /* not an object! */ + + item->type=cJSON_Object; + value=skip(value+1); + if (*value=='}') return value+1; /* empty array. */ + + item->child=child=cJSON_New_Item(); + if (!item->child) return 0; + value=skip(parse_string(child,skip(value))); + if (!value) return 0; + child->string=child->valuestring;child->valuestring=0; + if (*value!=':') {ep=value;return 0;} /* fail! */ + value=skip(parse_value(child,skip(value+1))); /* skip any spacing, get the value. */ + if (!value) return 0; + + while (*value==',') + { + cJSON *new_item; + if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */ + child->next=new_item;new_item->prev=child;child=new_item; + value=skip(parse_string(child,skip(value+1))); + if (!value) return 0; + child->string=child->valuestring;child->valuestring=0; + if (*value!=':') {ep=value;return 0;} /* fail! */ + value=skip(parse_value(child,skip(value+1))); /* skip any spacing, get the value. */ + if (!value) return 0; + } + + if (*value=='}') return value+1; /* end of array */ + ep=value;return 0; /* malformed. */ +} + +/* Render an object to text. */ +static char *print_object(cJSON *item,int depth,int fmt) +{ + char **entries=0,**names=0; + char *out=0,*ptr,*ret,*str;int len=7,i=0,j; + cJSON *child=item->child; + int numentries=0,fail=0; + /* Count the number of entries. */ + while (child) numentries++,child=child->next; + /* Allocate space for the names and the objects */ + entries=(char**)cJSON_malloc(numentries*sizeof(char*)); + if (!entries) return 0; + names=(char**)cJSON_malloc(numentries*sizeof(char*)); + if (!names) {cJSON_free(entries);return 0;} + memset(entries,0,sizeof(char*)*numentries); + memset(names,0,sizeof(char*)*numentries); + + /* Collect all the results into our arrays: */ + child=item->child;depth++;if (fmt) len+=depth; + while (child) + { + names[i]=str=print_string_ptr(child->string); + entries[i++]=ret=print_value(child,depth,fmt); + if (str && ret) len+=strlen(ret)+strlen(str)+2+(fmt?2+depth:0); else fail=1; + child=child->next; + } + + /* Try to allocate the output string */ + if (!fail) out=(char*)cJSON_malloc(len); + if (!out) fail=1; + + /* Handle failure */ + if (fail) + { + for (i=0;i<numentries;i++) {if (names[i]) cJSON_free(names[i]);if (entries[i]) cJSON_free(entries[i]);} + cJSON_free(names);cJSON_free(entries); + return 0; + } + + /* Compose the output: */ + *out='{';ptr=out+1;if (fmt)*ptr++='\n';*ptr=0; + for (i=0;i<numentries;i++) + { + if (fmt) for (j=0;j<depth;j++) *ptr++='\t'; + strcpy(ptr,names[i]);ptr+=strlen(names[i]); + *ptr++=':';if (fmt) *ptr++='\t'; + strcpy(ptr,entries[i]);ptr+=strlen(entries[i]); + if (i!=numentries-1) *ptr++=','; + if (fmt) *ptr++='\n';*ptr=0; + cJSON_free(names[i]);cJSON_free(entries[i]); + } + + cJSON_free(names);cJSON_free(entries); + if (fmt) for (i=0;i<depth-1;i++) *ptr++='\t'; + *ptr++='}';*ptr++=0; + return out; +} + +/* Get Array size/item / object item. */ +int cJSON_GetArraySize(cJSON *array) {cJSON *c=array->child;int i=0;while(c)i++,c=c->next;return i;} +cJSON *cJSON_GetArrayItem(cJSON *array,int item) {cJSON *c=array->child; while (c && item>0) item--,c=c->next; return c;} +cJSON *cJSON_GetObjectItem(cJSON *object,const char *string) {cJSON *c=object->child; while (c && cJSON_strcasecmp(c->string,string)) c=c->next; return c;} + +/* Utility for array list handling. */ +static void suffix_object(cJSON *prev,cJSON *item) {prev->next=item;item->prev=prev;} +/* Utility for handling references. */ +static cJSON *create_reference(cJSON *item) {cJSON *ref=cJSON_New_Item();if (!ref) return 0;memcpy(ref,item,sizeof(cJSON));ref->string=0;ref->type|=cJSON_IsReference;ref->next=ref->prev=0;return ref;} + +/* Add item to array/object. */ +void cJSON_AddItemToArray(cJSON *array, cJSON *item) {cJSON *c=array->child;if (!item) return; if (!c) {array->child=item;} else {while (c && c->next) c=c->next; suffix_object(c,item);}} +void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item) {if (!item) return; if (item->string) cJSON_free(item->string);item->string=cJSON_strdup(string);cJSON_AddItemToArray(object,item);} +void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) {cJSON_AddItemToArray(array,create_reference(item));} +void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item) {cJSON_AddItemToObject(object,string,create_reference(item));} + +cJSON *cJSON_DetachItemFromArray(cJSON *array,int which) {cJSON *c=array->child;while (c && which>0) c=c->next,which--;if (!c) return 0; + if (c->prev) c->prev->next=c->next;if (c->next) c->next->prev=c->prev;if (c==array->child) array->child=c->next;c->prev=c->next=0;return c;} +void cJSON_DeleteItemFromArray(cJSON *array,int which) {cJSON_Delete(cJSON_DetachItemFromArray(array,which));} +cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string) {int i=0;cJSON *c=object->child;while (c && cJSON_strcasecmp(c->string,string)) i++,c=c->next;if (c) return cJSON_DetachItemFromArray(object,i);return 0;} +void cJSON_DeleteItemFromObject(cJSON *object,const char *string) {cJSON_Delete(cJSON_DetachItemFromObject(object,string));} + +/* Replace array/object items with new ones. */ +void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem) {cJSON *c=array->child;while (c && which>0) c=c->next,which--;if (!c) return; + newitem->next=c->next;newitem->prev=c->prev;if (newitem->next) newitem->next->prev=newitem; + if (c==array->child) array->child=newitem; else newitem->prev->next=newitem;c->next=c->prev=0;cJSON_Delete(c);} +void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem){int i=0;cJSON *c=object->child;while(c && cJSON_strcasecmp(c->string,string))i++,c=c->next;if(c){newitem->string=cJSON_strdup(string);cJSON_ReplaceItemInArray(object,i,newitem);}} + +/* Create basic types: */ +cJSON *cJSON_CreateNull() {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_NULL;return item;} +cJSON *cJSON_CreateTrue() {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_True;return item;} +cJSON *cJSON_CreateFalse() {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_False;return item;} +cJSON *cJSON_CreateBool(int b) {cJSON *item=cJSON_New_Item();if(item)item->type=b?cJSON_True:cJSON_False;return item;} +cJSON *cJSON_CreateNumber(double num) {cJSON *item=cJSON_New_Item();if(item){item->type=cJSON_Number;item->valuedouble=num;item->valueint=(int)num;}return item;} +cJSON *cJSON_CreateString(const char *string) {cJSON *item=cJSON_New_Item();if(item){item->type=cJSON_String;item->valuestring=cJSON_strdup(string);}return item;} +cJSON *cJSON_CreateArray() {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_Array;return item;} +cJSON *cJSON_CreateObject() {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_Object;return item;} + +/* Create Arrays: */ +cJSON *cJSON_CreateIntArray(int *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && i<count;i++){n=cJSON_CreateNumber(numbers[i]);if(!i)a->child=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateFloatArray(float *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && i<count;i++){n=cJSON_CreateNumber(numbers[i]);if(!i)a->child=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateDoubleArray(double *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && i<count;i++){n=cJSON_CreateNumber(numbers[i]);if(!i)a->child=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateStringArray(const char **strings,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && i<count;i++){n=cJSON_CreateString(strings[i]);if(!i)a->child=n;else suffix_object(p,n);p=n;}return a;} diff --git a/plugins/omelasticsearch/cJSON/cjson.h b/plugins/omelasticsearch/cJSON/cjson.h new file mode 100644 index 00000000..a621720c --- /dev/null +++ b/plugins/omelasticsearch/cJSON/cjson.h @@ -0,0 +1,130 @@ +/* + Copyright (c) 2009 Dave Gamble + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#ifndef cJSON__h +#define cJSON__h + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* cJSON Types: */ +#define cJSON_False 0 +#define cJSON_True 1 +#define cJSON_NULL 2 +#define cJSON_Number 3 +#define cJSON_String 4 +#define cJSON_Array 5 +#define cJSON_Object 6 + +#define cJSON_IsReference 256 + +/* The cJSON structure: */ +typedef struct cJSON { + struct cJSON *next,*prev; /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ + struct cJSON *child; /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ + + int type; /* The type of the item, as above. */ + + char *valuestring; /* The item's string, if type==cJSON_String */ + int valueint; /* The item's number, if type==cJSON_Number */ + double valuedouble; /* The item's number, if type==cJSON_Number */ + + char *string; /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ +} cJSON; + +typedef struct cJSON_Hooks { + void *(*malloc_fn)(size_t sz); + void (*free_fn)(void *ptr); +} cJSON_Hooks; + +/* Supply malloc, realloc and free functions to cJSON */ +extern void cJSON_InitHooks(cJSON_Hooks* hooks); + + +/* Supply a block of JSON, and this returns a cJSON object you can interrogate. Call cJSON_Delete when finished. */ +extern cJSON *cJSON_Parse(const char *value); +/* Render a cJSON entity to text for transfer/storage. Free the char* when finished. */ +extern char *cJSON_Print(cJSON *item); +/* Render a cJSON entity to text for transfer/storage without any formatting. Free the char* when finished. */ +extern char *cJSON_PrintUnformatted(cJSON *item); +/* Delete a cJSON entity and all subentities. */ +extern void cJSON_Delete(cJSON *c); + +/* Returns the number of items in an array (or object). */ +extern int cJSON_GetArraySize(cJSON *array); +/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */ +extern cJSON *cJSON_GetArrayItem(cJSON *array,int item); +/* Get item "string" from object. Case insensitive. */ +extern cJSON *cJSON_GetObjectItem(cJSON *object,const char *string); + +/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ +extern const char *cJSON_GetErrorPtr(); + +/* These calls create a cJSON item of the appropriate type. */ +extern cJSON *cJSON_CreateNull(); +extern cJSON *cJSON_CreateTrue(); +extern cJSON *cJSON_CreateFalse(); +extern cJSON *cJSON_CreateBool(int b); +extern cJSON *cJSON_CreateNumber(double num); +extern cJSON *cJSON_CreateString(const char *string); +extern cJSON *cJSON_CreateArray(); +extern cJSON *cJSON_CreateObject(); + +/* These utilities create an Array of count items. */ +extern cJSON *cJSON_CreateIntArray(int *numbers,int count); +extern cJSON *cJSON_CreateFloatArray(float *numbers,int count); +extern cJSON *cJSON_CreateDoubleArray(double *numbers,int count); +extern cJSON *cJSON_CreateStringArray(const char **strings,int count); + +/* Append item to the specified array/object. */ +extern void cJSON_AddItemToArray(cJSON *array, cJSON *item); +extern void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item); +/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ +extern void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); +extern void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item); + +/* Remove/Detatch items from Arrays/Objects. */ +extern cJSON *cJSON_DetachItemFromArray(cJSON *array,int which); +extern void cJSON_DeleteItemFromArray(cJSON *array,int which); +extern cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string); +extern void cJSON_DeleteItemFromObject(cJSON *object,const char *string); + +/* Update array items. */ +extern void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem); +extern void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); + +/* rger: added helpers */ + +char *cJSON_print_number(cJSON *item); +#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull()) +#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue()) +#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse()) +#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n)) +#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s)) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/plugins/omelasticsearch/cJSON/test.c b/plugins/omelasticsearch/cJSON/test.c new file mode 100644 index 00000000..2cab632a --- /dev/null +++ b/plugins/omelasticsearch/cJSON/test.c @@ -0,0 +1,156 @@ +/* + Copyright (c) 2009 Dave Gamble + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#include <stdio.h> +#include <stdlib.h> +#include "cJSON.h" + +/* Parse text to JSON, then render back to text, and print! */ +void doit(char *text) +{ + char *out;cJSON *json; + + json=cJSON_Parse(text); + if (!json) {printf("Error before: [%s]\n",cJSON_GetErrorPtr());} + else + { + out=cJSON_Print(json); + cJSON_Delete(json); + printf("%s\n",out); + free(out); + } +} + +/* Read a file, parse, render back, etc. */ +void dofile(char *filename) +{ + FILE *f=fopen(filename,"rb");fseek(f,0,SEEK_END);long len=ftell(f);fseek(f,0,SEEK_SET); + char *data=malloc(len+1);fread(data,1,len,f);fclose(f); + doit(data); + free(data); +} + +/* Used by some code below as an example datatype. */ +struct record {const char *precision;double lat,lon;const char *address,*city,*state,*zip,*country; }; + +/* Create a bunch of objects as demonstration. */ +void create_objects() +{ + cJSON *root,*fmt,*img,*thm,*fld;char *out;int i; /* declare a few. */ + + /* Here we construct some JSON standards, from the JSON site. */ + + /* Our "Video" datatype: */ + root=cJSON_CreateObject(); + cJSON_AddItemToObject(root, "name", cJSON_CreateString("Jack (\"Bee\") Nimble")); + cJSON_AddItemToObject(root, "format", fmt=cJSON_CreateObject()); + cJSON_AddStringToObject(fmt,"type", "rect"); + cJSON_AddNumberToObject(fmt,"width", 1920); + cJSON_AddNumberToObject(fmt,"height", 1080); + cJSON_AddFalseToObject (fmt,"interlace"); + cJSON_AddNumberToObject(fmt,"frame rate", 24); + + out=cJSON_Print(root); cJSON_Delete(root); printf("%s\n",out); free(out); /* Print to text, Delete the cJSON, print it, release the string. + + /* Our "days of the week" array: */ + const char *strings[7]={"Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"}; + root=cJSON_CreateStringArray(strings,7); + + out=cJSON_Print(root); cJSON_Delete(root); printf("%s\n",out); free(out); + + /* Our matrix: */ + int numbers[3][3]={{0,-1,0},{1,0,0},{0,0,1}}; + root=cJSON_CreateArray(); + for (i=0;i<3;i++) cJSON_AddItemToArray(root,cJSON_CreateIntArray(numbers[i],3)); + +/* cJSON_ReplaceItemInArray(root,1,cJSON_CreateString("Replacement")); */ + + out=cJSON_Print(root); cJSON_Delete(root); printf("%s\n",out); free(out); + + + /* Our "gallery" item: */ + int ids[4]={116,943,234,38793}; + root=cJSON_CreateObject(); + cJSON_AddItemToObject(root, "Image", img=cJSON_CreateObject()); + cJSON_AddNumberToObject(img,"Width",800); + cJSON_AddNumberToObject(img,"Height",600); + cJSON_AddStringToObject(img,"Title","View from 15th Floor"); + cJSON_AddItemToObject(img, "Thumbnail", thm=cJSON_CreateObject()); + cJSON_AddStringToObject(thm, "Url", "http:/*www.example.com/image/481989943"); + cJSON_AddNumberToObject(thm,"Height",125); + cJSON_AddStringToObject(thm,"Width","100"); + cJSON_AddItemToObject(img,"IDs", cJSON_CreateIntArray(ids,4)); + + out=cJSON_Print(root); cJSON_Delete(root); printf("%s\n",out); free(out); + + /* Our array of "records": */ + struct record fields[2]={ + {"zip",37.7668,-1.223959e+2,"","SAN FRANCISCO","CA","94107","US"}, + {"zip",37.371991,-1.22026e+2,"","SUNNYVALE","CA","94085","US"}}; + + root=cJSON_CreateArray(); + for (i=0;i<2;i++) + { + cJSON_AddItemToArray(root,fld=cJSON_CreateObject()); + cJSON_AddStringToObject(fld, "precision", fields[i].precision); + cJSON_AddNumberToObject(fld, "Latitude", fields[i].lat); + cJSON_AddNumberToObject(fld, "Longitude", fields[i].lon); + cJSON_AddStringToObject(fld, "Address", fields[i].address); + cJSON_AddStringToObject(fld, "City", fields[i].city); + cJSON_AddStringToObject(fld, "State", fields[i].state); + cJSON_AddStringToObject(fld, "Zip", fields[i].zip); + cJSON_AddStringToObject(fld, "Country", fields[i].country); + } + +/* cJSON_ReplaceItemInObject(cJSON_GetArrayItem(root,1),"City",cJSON_CreateIntArray(ids,4)); */ + + out=cJSON_Print(root); cJSON_Delete(root); printf("%s\n",out); free(out); + +} + +int main (int argc, const char * argv[]) { + /* a bunch of json: */ + char text1[]="{\n\"name\": \"Jack (\\\"Bee\\\") Nimble\", \n\"format\": {\"type\": \"rect\", \n\"width\": 1920, \n\"height\": 1080, \n\"interlace\": false,\"frame rate\": 24\n}\n}"; + char text2[]="[\"Sunday\", \"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\", \"Friday\", \"Saturday\"]"; + char text3[]="[\n [0, -1, 0],\n [1, 0, 0],\n [0, 0, 1]\n ]\n"; + char text4[]="{\n \"Image\": {\n \"Width\": 800,\n \"Height\": 600,\n \"Title\": \"View from 15th Floor\",\n \"Thumbnail\": {\n \"Url\": \"http:/*www.example.com/image/481989943\",\n \"Height\": 125,\n \"Width\": \"100\"\n },\n \"IDs\": [116, 943, 234, 38793]\n }\n }"; + char text5[]="[\n {\n \"precision\": \"zip\",\n \"Latitude\": 37.7668,\n \"Longitude\": -122.3959,\n \"Address\": \"\",\n \"City\": \"SAN FRANCISCO\",\n \"State\": \"CA\",\n \"Zip\": \"94107\",\n \"Country\": \"US\"\n },\n {\n \"precision\": \"zip\",\n \"Latitude\": 37.371991,\n \"Longitude\": -122.026020,\n \"Address\": \"\",\n \"City\": \"SUNNYVALE\",\n \"State\": \"CA\",\n \"Zip\": \"94085\",\n \"Country\": \"US\"\n }\n ]"; + + /* Process each json textblock by parsing, then rebuilding: */ + doit(text1); + doit(text2); + doit(text3); + doit(text4); + doit(text5); + + /* Parse standard testfiles: +/* dofile("../../tests/test1"); */ +/* dofile("../../tests/test2"); */ +/* dofile("../../tests/test3"); */ +/* dofile("../../tests/test4"); */ +/* dofile("../../tests/test5"); */ + + /* Now some samplecode for building objects concisely: */ + create_objects(); + + return 0; +} diff --git a/plugins/omelasticsearch/cJSON/tests/test1 b/plugins/omelasticsearch/cJSON/tests/test1 new file mode 100644 index 00000000..eacfbf5e --- /dev/null +++ b/plugins/omelasticsearch/cJSON/tests/test1 @@ -0,0 +1,22 @@ +{ + "glossary": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } +} diff --git a/plugins/omelasticsearch/cJSON/tests/test2 b/plugins/omelasticsearch/cJSON/tests/test2 new file mode 100644 index 00000000..5600991a --- /dev/null +++ b/plugins/omelasticsearch/cJSON/tests/test2 @@ -0,0 +1,11 @@ +{"menu": { + "id": "file", + "value": "File", + "popup": { + "menuitem": [ + {"value": "New", "onclick": "CreateNewDoc()"}, + {"value": "Open", "onclick": "OpenDoc()"}, + {"value": "Close", "onclick": "CloseDoc()"} + ] + } +}} diff --git a/plugins/omelasticsearch/cJSON/tests/test3 b/plugins/omelasticsearch/cJSON/tests/test3 new file mode 100644 index 00000000..5662b377 --- /dev/null +++ b/plugins/omelasticsearch/cJSON/tests/test3 @@ -0,0 +1,26 @@ +{"widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "name": "sun1", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "name": "text1", + "hOffset": 250, + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } +}}
\ No newline at end of file diff --git a/plugins/omelasticsearch/cJSON/tests/test4 b/plugins/omelasticsearch/cJSON/tests/test4 new file mode 100644 index 00000000..d540b57f --- /dev/null +++ b/plugins/omelasticsearch/cJSON/tests/test4 @@ -0,0 +1,88 @@ +{"web-app": { + "servlet": [ + { + "servlet-name": "cofaxCDS", + "servlet-class": "org.cofax.cds.CDSServlet", + "init-param": { + "configGlossary:installationAt": "Philadelphia, PA", + "configGlossary:adminEmail": "ksm@pobox.com", + "configGlossary:poweredBy": "Cofax", + "configGlossary:poweredByIcon": "/images/cofax.gif", + "configGlossary:staticPath": "/content/static", + "templateProcessorClass": "org.cofax.WysiwygTemplate", + "templateLoaderClass": "org.cofax.FilesTemplateLoader", + "templatePath": "templates", + "templateOverridePath": "", + "defaultListTemplate": "listTemplate.htm", + "defaultFileTemplate": "articleTemplate.htm", + "useJSP": false, + "jspListTemplate": "listTemplate.jsp", + "jspFileTemplate": "articleTemplate.jsp", + "cachePackageTagsTrack": 200, + "cachePackageTagsStore": 200, + "cachePackageTagsRefresh": 60, + "cacheTemplatesTrack": 100, + "cacheTemplatesStore": 50, + "cacheTemplatesRefresh": 15, + "cachePagesTrack": 200, + "cachePagesStore": 100, + "cachePagesRefresh": 10, + "cachePagesDirtyRead": 10, + "searchEngineListTemplate": "forSearchEnginesList.htm", + "searchEngineFileTemplate": "forSearchEngines.htm", + "searchEngineRobotsDb": "WEB-INF/robots.db", + "useDataStore": true, + "dataStoreClass": "org.cofax.SqlDataStore", + "redirectionClass": "org.cofax.SqlRedirection", + "dataStoreName": "cofax", + "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", + "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", + "dataStoreUser": "sa", + "dataStorePassword": "dataStoreTestQuery", + "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", + "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", + "dataStoreInitConns": 10, + "dataStoreMaxConns": 100, + "dataStoreConnUsageLimit": 100, + "dataStoreLogLevel": "debug", + "maxUrlLength": 500}}, + { + "servlet-name": "cofaxEmail", + "servlet-class": "org.cofax.cds.EmailServlet", + "init-param": { + "mailHost": "mail1", + "mailHostOverride": "mail2"}}, + { + "servlet-name": "cofaxAdmin", + "servlet-class": "org.cofax.cds.AdminServlet"}, + + { + "servlet-name": "fileServlet", + "servlet-class": "org.cofax.cds.FileServlet"}, + { + "servlet-name": "cofaxTools", + "servlet-class": "org.cofax.cms.CofaxToolsServlet", + "init-param": { + "templatePath": "toolstemplates/", + "log": 1, + "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", + "logMaxSize": "", + "dataLog": 1, + "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", + "dataLogMaxSize": "", + "removePageCache": "/content/admin/remove?cache=pages&id=", + "removeTemplateCache": "/content/admin/remove?cache=templates&id=", + "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", + "lookInContext": 1, + "adminGroupID": 4, + "betaServer": true}}], + "servlet-mapping": { + "cofaxCDS": "/", + "cofaxEmail": "/cofaxutil/aemail/*", + "cofaxAdmin": "/admin/*", + "fileServlet": "/static/*", + "cofaxTools": "/tools/*"}, + + "taglib": { + "taglib-uri": "cofax.tld", + "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}
\ No newline at end of file diff --git a/plugins/omelasticsearch/cJSON/tests/test5 b/plugins/omelasticsearch/cJSON/tests/test5 new file mode 100644 index 00000000..49980ca2 --- /dev/null +++ b/plugins/omelasticsearch/cJSON/tests/test5 @@ -0,0 +1,27 @@ +{"menu": { + "header": "SVG Viewer", + "items": [ + {"id": "Open"}, + {"id": "OpenNew", "label": "Open New"}, + null, + {"id": "ZoomIn", "label": "Zoom In"}, + {"id": "ZoomOut", "label": "Zoom Out"}, + {"id": "OriginalView", "label": "Original View"}, + null, + {"id": "Quality"}, + {"id": "Pause"}, + {"id": "Mute"}, + null, + {"id": "Find", "label": "Find..."}, + {"id": "FindAgain", "label": "Find Again"}, + {"id": "Copy"}, + {"id": "CopyAgain", "label": "Copy Again"}, + {"id": "CopySVG", "label": "Copy SVG"}, + {"id": "ViewSVG", "label": "View SVG"}, + {"id": "ViewSource", "label": "View Source"}, + {"id": "SaveAs", "label": "Save As"}, + null, + {"id": "Help"}, + {"id": "About", "label": "About Adobe CVG Viewer..."} + ] +}} diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index f77caeca..a5833b15 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -34,6 +34,10 @@ #include <signal.h> #include <errno.h> #include <time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "cJSON/cjson.h" #include "conf.h" #include "syslogd-types.h" #include "srUtils.h" @@ -64,8 +68,10 @@ STATSCOUNTER_DEF(indexSuccess, mutIndexSuccess) */ typedef struct curl_slist HEADER; typedef struct _instanceData { - uchar *server; int port; + int replyLen; + int fdErrFile; /* error file fd or -1 if not open */ + uchar *server; uchar *uid; uchar *pwd; uchar *searchIndex; @@ -73,6 +79,9 @@ typedef struct _instanceData { uchar *parent; uchar *tplName; uchar *timeout; + uchar *restURL; /* last used URL for error reporting */ + uchar *errorFile; + char *reply; sbool dynSrchIdx; sbool dynSrchType; sbool dynParent; @@ -104,6 +113,7 @@ static struct cnfparamdescr actpdescr[] = { { "bulkmode", eCmdHdlrBinary, 0 }, { "asyncrepl", eCmdHdlrBinary, 0 }, { "timeout", eCmdHdlrGetWord, 0 }, + { "errorfile", eCmdHdlrGetWord, 0 }, { "template", eCmdHdlrGetWord, 1 } }; static struct cnfparamblk actpblk = @@ -114,6 +124,8 @@ static struct cnfparamblk actpblk = BEGINcreateInstance CODESTARTcreateInstance + pData->restURL = NULL; + pData->fdErrFile = -1; ENDcreateInstance BEGINisCompatibleWithFeature @@ -132,6 +144,8 @@ CODESTARTfreeInstance curl_easy_cleanup(pData->curlHandle); pData->curlHandle = NULL; } + if(pData->fdErrFile != -1) + close(pData->fdErrFile); free(pData->server); free(pData->uid); free(pData->pwd); @@ -139,6 +153,9 @@ CODESTARTfreeInstance free(pData->searchType); free(pData->parent); free(pData->tplName); + free(pData->timeout); + free(pData->restURL); + free(pData->errorFile); ENDfreeInstance BEGINdbgPrintInstInfo @@ -158,6 +175,8 @@ CODESTARTdbgPrintInstInfo dbgprintf("\tdynamic parent=%d\n", pData->dynParent); dbgprintf("\tasync replication=%d\n", pData->asyncRepl); dbgprintf("\tbulkmode=%d\n", pData->bulkmode); + dbgprintf("\terrorfile='%s'\n", pData->errorFile == NULL ? + (uchar*)"(not configured)" : pData->errorFile); ENDdbgPrintInstInfo @@ -202,12 +221,16 @@ checkConn(instanceData *pData) curl_easy_setopt(curl, CURLOPT_URL, cstr); free(cstr); + pData->reply = NULL; + pData->replyLen = 0; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, pData); res = curl_easy_perform(curl); if(res != CURLE_OK) { DBGPRINTF("omelasticsearch: checkConn() curl_easy_perform() " "failed: %s\n", curl_easy_strerror(res)); ABORT_FINALIZE(RS_RET_SUSPENDED); } + free(pData->reply); DBGPRINTF("omelasticsearch: checkConn() completed with success\n"); finalize_it: @@ -271,7 +294,6 @@ static rsRetVal setCurlURL(instanceData *pData, uchar **tpls) { char authBuf[1024]; - char *restURL; uchar *searchIndex; uchar *searchType; uchar *parent; @@ -305,11 +327,12 @@ setCurlURL(instanceData *pData, uchar **tpls) if(r == 0) r = es_addBuf(&url, "parent=", sizeof("parent=")-1); if(r == 0) r = es_addBuf(&url, (char*)parent, ustrlen(parent)); } - restURL = es_str2cstr(url, NULL); - curl_easy_setopt(pData->curlHandle, CURLOPT_URL, restURL); + + free(pData->restURL); + pData->restURL = (uchar*)es_str2cstr(url, NULL); + curl_easy_setopt(pData->curlHandle, CURLOPT_URL, pData->restURL); es_deleteStr(url); - DBGPRINTF("omelasticsearch: using REST URL: '%s'\n", restURL); - free(restURL); + DBGPRINTF("omelasticsearch: using REST URL: '%s'\n", pData->restURL); if(pData->uid != NULL) { rLocal = snprintf(authBuf, sizeof(authBuf), "%s:%s", pData->uid, @@ -347,9 +370,6 @@ buildBatch(instanceData *pData, uchar *message, uchar **tpls) # define META_END "\"}}\n" getIndexTypeAndParent(pData, tpls, &searchIndex, &searchType, &parent); -dbgprintf("AAA: searchIndex: '%s'\n", searchIndex); -dbgprintf("AAA: searchType: '%s'\n", searchType); -dbgprintf("AAA: parent: '%s'\n", parent); r = es_addBuf(&pData->batch.data, META_STRT, sizeof(META_STRT)-1); if(r == 0) r = es_addBuf(&pData->batch.data, (char*)searchIndex, ustrlen(searchIndex)); @@ -373,22 +393,158 @@ finalize_it: RETiRet; } + +/* write data error request/replies to separate error file + * Note: we open the file but never close it before exit. If it + * needs to be closed, HUP must be sent. + */ +static inline rsRetVal +writeDataError(instanceData *pData, cJSON **pReplyRoot, uchar *reqmsg) +{ + char *rendered = NULL; + cJSON *errRoot; + cJSON *req; + cJSON *replyRoot = *pReplyRoot; + char errStr[1024]; + DEFiRet; + + if(pData->errorFile == NULL) { + DBGPRINTF("omelasticsearch: no local error logger defined - " + "ignoring ES error information\n"); + FINALIZE; + } + + if(pData->fdErrFile == -1) { + pData->fdErrFile = open((char*)pData->errorFile, + O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE|O_CLOEXEC, + S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP); + if(pData->fdErrFile == -1) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + DBGPRINTF("omelasticsearch: error opening error file: %s\n", errStr); + ABORT_FINALIZE(RS_RET_ERR); + } + } + if((req=cJSON_CreateObject()) == NULL) ABORT_FINALIZE(RS_RET_ERR); + cJSON_AddItemToObject(req, "url", cJSON_CreateString((char*)pData->restURL)); + cJSON_AddItemToObject(req, "postdata", cJSON_CreateString((char*)reqmsg)); + + if((errRoot=cJSON_CreateObject()) == NULL) ABORT_FINALIZE(RS_RET_ERR); + cJSON_AddItemToObject(errRoot, "request", req); + cJSON_AddItemToObject(errRoot, "reply", replyRoot); + rendered = cJSON_Print(errRoot); +DBGPRINTF("omelasticsearch: error record: '%s'\n", rendered); + write(pData->fdErrFile, rendered, strlen(rendered)); + free(rendered); + cJSON_Delete(errRoot); + *pReplyRoot = NULL; /* tell caller not to delete once again! */ + +finalize_it: + if(rendered != NULL) + free(rendered); + RETiRet; +} + + +static inline rsRetVal +checkResultBulkmode(instanceData *pData, cJSON *root) +{ + int i; + int numitems; + cJSON *items; + cJSON *item; + cJSON *create; + cJSON *ok; + DEFiRet; + + items = cJSON_GetObjectItem(root, "items"); + if(items == NULL || items->type != cJSON_Array) { + DBGPRINTF("omelasticsearch: error in elasticsearch reply: " + "bulkmode insert does not return array, reply is: %s\n", + pData->reply); + ABORT_FINALIZE(RS_RET_DATAFAIL); + } + numitems = cJSON_GetArraySize(items); +DBGPRINTF("omelasticsearch: %d items in reply\n", numitems); + for(i = 0 ; i < numitems ; ++i) { + item = cJSON_GetArrayItem(items, i); + if(item == NULL) { + DBGPRINTF("omelasticsearch: error in elasticsearch reply: " + "cannot obtain reply array item %d\n", i); + ABORT_FINALIZE(RS_RET_DATAFAIL); + } + create = cJSON_GetObjectItem(item, "create"); + if(create == NULL || create->type != cJSON_Object) { + DBGPRINTF("omelasticsearch: error in elasticsearch reply: " + "cannot obtain 'create' item for #%d\n", i); + ABORT_FINALIZE(RS_RET_DATAFAIL); + } + ok = cJSON_GetObjectItem(create, "ok"); + if(ok == NULL || ok->type != cJSON_True) { + DBGPRINTF("omelasticsearch: error in elasticsearch reply: " + "item %d, prop ok (%p) not ok\n", i, ok); + ABORT_FINALIZE(RS_RET_DATAFAIL); + } + } + +finalize_it: + RETiRet; +} + + +static inline rsRetVal +checkResult(instanceData *pData, uchar *reqmsg) +{ + cJSON *root; + cJSON *ok; + DEFiRet; + + root = cJSON_Parse(pData->reply); + if(root == NULL) { + DBGPRINTF("omelasticsearch: could not parse JSON result \n"); + ABORT_FINALIZE(RS_RET_ERR); + } + + if(pData->bulkmode) { + iRet = checkResultBulkmode(pData, root); + } else { + ok = cJSON_GetObjectItem(root, "ok"); + if(ok == NULL || ok->type != cJSON_True) { + iRet = RS_RET_DATAFAIL; + } + } + + /* Note: we ignore errors writing the error file, as we cannot handle + * these in any case. + */ + if(iRet == RS_RET_DATAFAIL) { + writeDataError(pData, &root, reqmsg); + iRet = RS_RET_OK; /* we have handled the problem! */ + } + +finalize_it: + if(root != NULL) + cJSON_Delete(root); + RETiRet; +} + + static rsRetVal -curlPost(instanceData *instance, uchar *message, int msglen, uchar **tpls) +curlPost(instanceData *pData, uchar *message, int msglen, uchar **tpls) { CURLcode code; - CURL *curl = instance->curlHandle; + CURL *curl = pData->curlHandle; DEFiRet; - if(instance->dynSrchIdx || instance->dynSrchType || instance->dynParent) - CHKiRet(setCurlURL(instance, tpls)); + pData->reply = NULL; + pData->replyLen = 0; + + if(pData->dynSrchIdx || pData->dynSrchType || pData->dynParent) + CHKiRet(setCurlURL(pData, tpls)); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (char *)message); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, pData); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, msglen); -dbgprintf("omelasticsearch: do curl_easy_perform()\n"); code = curl_easy_perform(curl); -DBGPRINTF("omelasticsearch: curl_easy_perform() returned %lld\n", (long long) code); switch (code) { case CURLE_COULDNT_RESOLVE_HOST: case CURLE_COULDNT_RESOLVE_PROXY: @@ -398,12 +554,18 @@ DBGPRINTF("omelasticsearch: curl_easy_perform() returned %lld\n", (long long) co DBGPRINTF("omelasticsearch: we are suspending ourselfs due " "to failure %lld of curl_easy_perform()\n", (long long) code); - return RS_RET_SUSPENDED; + ABORT_FINALIZE(RS_RET_SUSPENDED); default: STATSCOUNTER_INC(indexSubmit, mutIndexSubmit); - return RS_RET_OK; + break; } + + pData->reply[pData->replyLen] = '\0'; /* byte has been reserved in malloc */ + DBGPRINTF("omelasticsearch: es reply: '%s'\n", pData->reply); + + CHKiRet(checkResult(pData, message)); finalize_it: + free(pData->reply); RETiRet; } @@ -424,7 +586,6 @@ CODESTARTdoAction if(pData->bulkmode) { CHKiRet(buildBatch(pData, ppString[0], ppString)); } else { -dbgprintf("omelasticsearch: doAction calling curlPost\n"); CHKiRet(curlPost(pData, ppString[0], strlen((char*)ppString[0]), ppString)); } @@ -449,35 +610,20 @@ ENDendTransaction size_t curlResult(void *ptr, size_t size, size_t nmemb, void *userdata) { - unsigned int i; char *p = (char *)ptr; - char *jsonData = (char *)userdata; - static char ok[] = "{\"ok\":true,"; - - ASSERT(size == 1); -DBGPRINTF("omelasticsearch request: %s\n", jsonData); -DBGPRINTF("omelasticsearch result: "); -for (i = 0; i < nmemb; i++) - DBGPRINTF("%c", p[i]); -DBGPRINTF("\n"); - - if (size == 1 && - nmemb > sizeof(ok)-1 && - strncmp(p, ok, sizeof(ok)-1) == 0) { - STATSCOUNTER_INC(indexSuccess, mutIndexSuccess); -dbgprintf("omelasticsearch ok\n"); - } else { -dbgprintf("omelasticsearch fail\n"); - STATSCOUNTER_INC(indexFailed, mutIndexFailed); - if (Debug) { - DBGPRINTF("omelasticsearch (fail) request: %s\n", jsonData); - DBGPRINTF("omelasticsearch (fail) result: "); - for (i = 0; i < nmemb; i++) - DBGPRINTF("%c", p[i]); - DBGPRINTF("\n"); - } + instanceData *pData = (instanceData*) userdata; + char *buf; + size_t newlen; + + newlen = pData->replyLen + size*nmemb; + if((buf = realloc(pData->reply, newlen + 1)) == NULL) { + DBGPRINTF("omelasticsearch: realloc failed in curlResult\n"); + return 0; /* abort due to failure */ } - return size * nmemb; + memcpy(buf+pData->replyLen, p, size*nmemb); + pData->replyLen = newlen; + pData->reply = buf; + return size*nmemb; } @@ -533,6 +679,7 @@ setInstParamDefaults(instanceData *pData) pData->asyncRepl = 0; pData->bulkmode = 0; pData->tplName = NULL; + pData->errorFile = NULL; } BEGINnewActInst @@ -552,6 +699,8 @@ CODESTARTnewActInst continue; if(!strcmp(actpblk.descr[i].name, "server")) { pData->server = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "errorfile")) { + pData->errorFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "serverport")) { pData->port = (int) pvals[i].val.d.n, NULL; } else if(!strcmp(actpblk.descr[i].name, "uid")) { @@ -693,6 +842,14 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct +BEGINdoHUP +CODESTARTdoHUP + if(pData->fdErrFile != -1) { + close(pData->fdErrFile); + pData->fdErrFile = -1; + } +ENDdoHUP + BEGINmodExit CODESTARTmodExit @@ -707,6 +864,7 @@ CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES +CODEqueryEtryPt_doHUP CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ ENDqueryEtryPt diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 7abbc258..fbc92d9c 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -72,7 +72,6 @@ librsyslog_la_SOURCES = \ sd-daemon.c \ sd-daemon.h \ \ - \ ../action.h \ ../action.c \ ../threads.c \ diff --git a/runtime/rsconf.c b/runtime/rsconf.c index cf29c720..0fc67499 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -67,6 +67,7 @@ #include "dirty.h" #include "template.h" +extern char* yytext; /* static data */ DEFobjStaticHelpers DEFobjCurrIf(ruleset) @@ -390,7 +391,7 @@ parser_errmsg(char *fmt, ...) int yyerror(char *s) { - parser_errmsg("%s", s); + parser_errmsg("%s on token '%s'", s, yytext); return 0; } void cnfDoObj(struct cnfobj *o) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index c02db53c..0bf0642a 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -377,6 +377,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_LEGA_ACT_NOT_SUPPORTED = -2215, /**< the module (no longer) supports legacy action syntax */ RS_RET_MAX_OMSR_REACHED = -2216, /**< max nbr of string requests reached, not supported by core */ RS_RET_UID_MISSING = -2217, /**< a user id is missing (but e.g. a password provided) */ + RS_RET_DATAFAIL = -2218, /**< data passed to action caused failure */ /* reserved for pre-v6.5 */ RS_RET_DUP_PARAM = -2220, /**< config parameter is given more than once */ RS_RET_MODULE_ALREADY_IN_CONF = -2221, /**< module already in current configuration */ diff --git a/runtime/stream.c b/runtime/stream.c index aee4d2ad..b8dd5150 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1134,6 +1134,7 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) DEFiRet; ISOBJ_TYPE_assert(pThis, strm); + DBGPRINTF("strmPhysWrite, stream %p, len %d\n", pThis, (int) lenBuf); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); diff --git a/tests/Makefile.am b/tests/Makefile.am index 715c024d..5bcaea75 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ if ENABLE_TESTBENCH # TODO: reenable TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller syslog_inject inputfilegen +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr syslog_caller syslog_inject inputfilegen minitcpsrv TESTS = $(TESTRUNS) #TESTS = $(TESTRUNS) cfg.sh @@ -70,6 +70,7 @@ TESTS += \ rscript_stop2.sh \ rscript_prifilt.sh \ rscript_optimizer1.sh \ + rscript_ruleset_call.sh \ cee_simple.sh \ cee_diskqueue.sh \ linkedlistqueue.sh @@ -284,6 +285,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rscript_prifilt.conf \ rscript_optimizer1.sh \ testsuites/rscript_optimizer1.conf \ + rscript_ruleset_call.sh \ + testsuites/rscript_ruleset_call.conf \ cee_simple.sh \ testsuites/cee_simple.conf \ cee_diskqueue.sh \ @@ -516,6 +519,9 @@ if ENABLE_GNUTLS tcpflood_LDADD += -lgcrypt endif +minitcpsrv_SOURCES = minitcpsrvr.c +minitcpsrv_LDADD = $(SOL_LIBS) + syslog_caller_SOURCES = syslog_caller.c syslog_caller_LDADD = $(SOL_LIBS) diff --git a/tests/minitcpsrvr.c b/tests/minitcpsrvr.c new file mode 100644 index 00000000..76dae0a5 --- /dev/null +++ b/tests/minitcpsrvr.c @@ -0,0 +1,61 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <fcntl.h> +#include <unistd.h> +#include <arpa/inet.h> + +static void +errout(char *reason) +{ + perror(reason); + exit(1); +} + +int +main(int argc, char *argv[]) +{ + int fds; + int fdc; + int fdf; + struct sockaddr_in srvAddr; + struct sockaddr_in cliAddr; + unsigned int srvAddrLen; + unsigned int cliAddrLen; + char wrkBuf[4096]; + ssize_t nRead; + + if(argc != 4) { + fprintf(stderr, "usage: minitcpsrvr ip-addr port outfile\n"); + exit(1); + } + + if(!strcmp(argv[3], "-")) { + fdf = 1; + } else { + fdf = open(argv[3], O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR|S_IWUSR); + if(fdf == -1) errout(argv[3]); + } + + fds = socket(AF_INET, SOCK_STREAM, 0); + srvAddr.sin_family = AF_INET; + srvAddr.sin_addr.s_addr = inet_addr(argv[1]); + srvAddr.sin_port = htons(atoi(argv[2])); + srvAddrLen = sizeof(srvAddr); + if(bind(fds, (struct sockaddr *)&srvAddr, srvAddrLen) != 0) + errout("bind"); + if(listen(fds, 20) != 0) errout("listen"); + cliAddrLen = sizeof(cliAddr); + + fdc = accept(fds, (struct sockaddr *)&cliAddr, &cliAddrLen); + while(1) { + nRead = read(fdc, wrkBuf, sizeof(wrkBuf)); + if(nRead == 0) break; + if(write(fdf, wrkBuf, nRead) != nRead) + errout("write"); + } + /* let the OS do the cleanup */ + return 0; +} diff --git a/tests/rscript_ruleset_call.sh b/tests/rscript_ruleset_call.sh new file mode 100755 index 00000000..e29f21da --- /dev/null +++ b/tests/rscript_ruleset_call.sh @@ -0,0 +1,13 @@ +# added 2012-10-29 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_ruleset_call.sh\]: testing rainerscript ruleset\(\) and call statement +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rscript_ruleset_call.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/tcp_forwarding_tpl.sh b/tests/tcp_forwarding_tpl.sh new file mode 100755 index 00000000..61114507 --- /dev/null +++ b/tests/tcp_forwarding_tpl.sh @@ -0,0 +1,30 @@ +# This test tests tcp forwarding with assigned template. To do so, a simple +# tcp listener service is started. +# added 2012-10-30 by Rgerhards. Released under GNU GPLv3+ +echo =============================================================================== +echo \[tcp_forwarding_tpl.sh\]: test for tcp forwarding with assigned template + +# create the pipe and start a background process that copies data from +# it to the "regular" work file +source $srcdir/diag.sh init +./minitcpsrvr 127.0.0.1 13514 rsyslog.out.log & +BGPROCESS=$! +echo background minitcpsrvr process id is $BGPROCESS + +# now do the usual run +source $srcdir/diag.sh startup tcp_forwarding_tpl.conf +# 10000 messages should be enough +source $srcdir/diag.sh injectmsg 0 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown + +# note: minitcpsrvr shuts down automatically if the connection is closed! +# (we still leave the code here in in case we need it later) +#echo shutting down minitcpsrv... +#kill $BGPROCESS +#wait $BGPROCESS +#echo background process has terminated, continue test... + +# and continue the usual checks +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/complex1.conf b/tests/testsuites/complex1.conf index 9e2441d4..9b6a9f35 100644 --- a/tests/testsuites/complex1.conf +++ b/tests/testsuites/complex1.conf @@ -3,6 +3,8 @@ $MaxMessageSize 10k $IncludeConfig diag-common.conf +$MainMsgQueueTimeoutEnqueue 5000 + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 @@ -13,6 +15,7 @@ $template dynfile,"rsyslog.out.%inputname%.%msg:F,58:2%.log" $Ruleset R13514 # queue params: $ActionQueueTimeoutShutdown 60000 +$ActionQueueTimeoutEnqueue 5000 $ActionQueueSize 5000 $ActionQueueSaveOnShutdown on $ActionQueueHighWaterMark 4900 @@ -36,6 +39,7 @@ $InputTCPServerRun 13514 $Ruleset R13515 # queue params: $ActionQueueTimeoutShutdown 60000 +$ActionQueueTimeoutEnqueue 5000 $ActionQueueSize 5000 $ActionQueueSaveOnShutdown on $ActionQueueHighWaterMark 4900 @@ -60,6 +64,7 @@ $InputTCPServerRun 13515 $Ruleset R13516 # queue params: $ActionQueueTimeoutShutdown 60000 +$ActionQueueTimeoutEnqueue 5000 $ActionQueueSize 5000 $ActionQueueSaveOnShutdown on $ActionQueueHighWaterMark 4900 diff --git a/tests/testsuites/rscript_ruleset_call.conf b/tests/testsuites/rscript_ruleset_call.conf new file mode 100644 index 00000000..96eab293 --- /dev/null +++ b/tests/testsuites/rscript_ruleset_call.conf @@ -0,0 +1,22 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="msg" field.delimiter="58" field.number="2") + constant(value="\n") +} + + +# we deliberately include continue/stop to make sure we have more than +# one statement. This catches grammar erorrs +ruleset(name="rs2") { + continue + action(type="omfile" file="./rsyslog.out.log" template="outfmt") + stop +} + +# this time we make sure a single statement is properly supported +ruleset(name="rs1") { + call rs2 +} + +if $msg contains 'msgnum' then call rs1 diff --git a/tests/testsuites/tcp_forwarding_tpl.conf b/tests/testsuites/tcp_forwarding_tpl.conf new file mode 100644 index 00000000..686c73ac --- /dev/null +++ b/tests/testsuites/tcp_forwarding_tpl.conf @@ -0,0 +1,7 @@ +$IncludeConfig diag-common.conf +$MainMsgQueueTimeoutShutdown 10000 +template(name="outfmt" type="string" string="%msg:F,58:2%\n") + +if $msg contains "msgnum:" then + action(type="omfwd" template="outfmt" + target="127.0.0.1" port="13514" protocol="tcp") diff --git a/tools/omfwd.c b/tools/omfwd.c index 2fd24bdf..129392d2 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -138,6 +138,7 @@ static struct cnfparamdescr actpdescr[] = { { "streamdriverauthmode", eCmdHdlrGetWord, 0 }, { "streamdriverpermittedpeers", eCmdHdlrGetWord, 0 }, { "resendlastmsgonreconnect", eCmdHdlrBinary, 0 }, + { "template", eCmdHdlrGetWord, 0 }, }; static struct cnfparamblk actpblk = { CNFPARAMBLK_VERSION, @@ -760,6 +761,7 @@ setInstParamDefaults(instanceData *pData) BEGINnewActInst struct cnfparamvals *pvals; + uchar *tplToUse; int i; rsRetVal localRet; CODESTARTnewActInst @@ -881,7 +883,8 @@ CODESTARTnewActInst } CODE_STD_STRING_REQUESTnewActInst(1) - CHKiRet(OMSRsetEntry(*ppOMSR, 0, ustrdup(getDfltTpl()), OMSR_NO_RQD_TPL_OPTS)); + tplToUse = ustrdup((pData->tplName == NULL) ? getDfltTpl() : pData->tplName); + CHKiRet(OMSRsetEntry(*ppOMSR, 0, tplToUse, OMSR_NO_RQD_TPL_OPTS)); CHKiRet(initTCP(pData)); CODE_STD_FINALIZERnewActInst diff --git a/tools/rsyslogd.8 b/tools/rsyslogd.8 index 86db690e..9ded4b9b 100644 --- a/tools/rsyslogd.8 +++ b/tools/rsyslogd.8 @@ -10,6 +10,7 @@ rsyslogd \- reliable and extended syslogd .RB [ " \-6 " ] .RB [ " \-A " ] .RB [ " \-d " ] +.RB [ " \-D " ] .RB [ " \-f " .I config file ] @@ -124,6 +125,11 @@ This option has been obsoleted and has no function any longer. It is still accepted in order not to break existing scripts. However, future versions may not support it. .TP +.B "\-D" +Runs the Bison config parser in debug mode. This may help when hard to find +syntax errors are reported. Please note that the output generated is deeply +.TP +technical and orignally targeted towards developers. .B "\-d" Turns on debug mode. Using this the daemon will not proceed a .BR fork (2) diff --git a/tools/syslogd.c b/tools/syslogd.c index d66bfb81..e347794b 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -92,6 +92,8 @@ #include <zlib.h> #endif +extern int yydebug; /* interface to flex */ + #include <netdb.h> #include "pidfile.h" @@ -1753,7 +1755,7 @@ int realMain(int argc, char **argv) * of other options, we do this during the inital option processing. * rgerhards, 2008-04-04 */ - while((ch = getopt(argc, argv, "46a:Ac:def:g:hi:l:m:M:nN:op:qQr::s:t:T:u:vwx")) != EOF) { + while((ch = getopt(argc, argv, "46a:Ac:dDef:g:hi:l:m:M:nN:op:qQr::s:t:T:u:vwx")) != EOF) { switch((char)ch) { case '4': case '6': @@ -1786,6 +1788,10 @@ int realMain(int argc, char **argv) case 'd': /* debug - must be handled now, so that debug is active during init! */ debugging_on = 1; Debug = 1; + yydebug = 1; + break; + case 'D': /* BISON debug */ + yydebug = 1; break; case 'e': /* log every message (no repeat message supression) */ bEOptionWasGiven = 1; |