From 904c01bb8c2fb277f50e31e097dff284a35332b8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Jul 2013 15:18:05 +0200 Subject: add lookup table sekeleton to build system --- runtime/Makefile.am | 2 ++ runtime/lookup.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/lookup.h | 33 +++++++++++++++++++++++ runtime/typedefs.h | 1 + 4 files changed, 111 insertions(+) create mode 100644 runtime/lookup.c create mode 100644 runtime/lookup.h diff --git a/runtime/Makefile.am b/runtime/Makefile.am index dea06fe0..510368a1 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -69,6 +69,8 @@ librsyslog_la_SOURCES = \ prop.h \ ratelimit.c \ ratelimit.h \ + lookup.c \ + lookup.h \ cfsysline.c \ cfsysline.h \ sd-daemon.c \ diff --git a/runtime/lookup.c b/runtime/lookup.c new file mode 100644 index 00000000..167370f3 --- /dev/null +++ b/runtime/lookup.c @@ -0,0 +1,75 @@ +/* lookup.c + * Support for lookup tables in RainerScript. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include +#include +#include + +#include "rsyslog.h" +#include "errmsg.h" +#include "lookup.h" +#include "msg.h" +#include "rsconf.h" +#include "dirty.h" + +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + +/* static data */ + +rsRetVal +lookupNew(lookup_t **ppThis, char *modname, char *dynname) +{ + lookup_t *pThis; + DEFiRet; + + CHKmalloc(pThis = calloc(1, sizeof(lookup_t))); + + *ppThis = pThis; +finalize_it: + RETiRet; +} +void +lookupDestruct(lookup_t *lookup) +{ + free(lookup); +} + +void +lookupModExit(void) +{ + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); +} + +rsRetVal +lookupModInit(void) +{ + DEFiRet; + CHKiRet(objGetObjInterface(&obj)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +finalize_it: + RETiRet; +} diff --git a/runtime/lookup.h b/runtime/lookup.h new file mode 100644 index 00000000..5eecf215 --- /dev/null +++ b/runtime/lookup.h @@ -0,0 +1,33 @@ +/* header for lookup.c + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_LOOKUP_H +#define INCLUDED_LOOKUP_H + +struct lookup_s { +}; + +/* prototypes */ +rsRetVal lookupNew(lookup_t **ppThis, char *modname, char *dynname); +void lookupDestruct(lookup_t *pThis); +rsRetVal lookupModInit(void); +void lookupModExit(void); + +#endif /* #ifndef INCLUDED_LOOKUP_H */ diff --git a/runtime/typedefs.h b/runtime/typedefs.h index d3f68b4a..616b60c9 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -100,6 +100,7 @@ typedef struct outchannels_s outchannels_t; typedef struct modConfData_s modConfData_t; typedef struct instanceConf_s instanceConf_t; typedef struct ratelimit_s ratelimit_t; +typedef struct lookup_s lookup_t; typedef struct action_s action_t; typedef int rs_size_t; /* we do never need more than 2Gig strings, signed permits to * use -1 as a special flag. */ -- cgit v1.2.3 From 9cf45a63f50b570ab5b82d379a62f885afb53861 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Jul 2013 09:30:58 +0200 Subject: milestone: add lookup_table stmt to RainerScript ... but still does nothing execpt calling the handler. --- grammar/lexer.l | 2 ++ grammar/rainerscript.h | 4 ++++ runtime/lookup.c | 37 +++++++++++++++++++++++++++++++++++-- runtime/lookup.h | 5 +++-- runtime/rsconf.c | 5 ++++- runtime/rsyslog.c | 2 ++ 6 files changed, 50 insertions(+), 5 deletions(-) diff --git a/grammar/lexer.l b/grammar/lexer.l index 237eb2a6..9678a8f0 100644 --- a/grammar/lexer.l +++ b/grammar/lexer.l @@ -188,6 +188,8 @@ int fileno(FILE *stream); BEGIN INOBJ; return BEGINOBJ; } "module"[ \n\t]*"(" { yylval.objType = CNFOBJ_MODULE; BEGIN INOBJ; return BEGINOBJ; } +"lookup_table"[ \n\t]*"(" { yylval.objType = CNFOBJ_LOOKUP_TABLE; + BEGIN INOBJ; return BEGINOBJ; } "action"[ \n\t]*"(" { BEGIN INOBJ; return BEGIN_ACTION; } ^[ \t]*:\$?[a-z\-]+[ ]*,[ ]*!?[a-z]+[ ]*,[ ]*\"(\\\"|[^\"])*\" { yylval.s = strdup(rmLeadingSpace(yytext)); diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h index 31b2eb93..cb8c968f 100644 --- a/grammar/rainerscript.h +++ b/grammar/rainerscript.h @@ -24,6 +24,7 @@ enum cnfobjType { CNFOBJ_TPL, CNFOBJ_PROPERTY, CNFOBJ_CONSTANT, + CNFOBJ_LOOKUP_TABLE, CNFOBJ_INVALID = 0 }; @@ -55,6 +56,9 @@ cnfobjType2str(enum cnfobjType ot) case CNFOBJ_CONSTANT: return "constant"; break; + case CNFOBJ_LOOKUP_TABLE: + return "lookup_table"; + break; default:return "error: invalid cnfobjType"; } } diff --git a/runtime/lookup.c b/runtime/lookup.c index 167370f3..7cb27b07 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -37,6 +37,16 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) /* static data */ +/* tables for interfacing with the v6 config system (as far as we need to) */ +static struct cnfparamdescr modpdescr[] = { + { "name", eCmdHdlrString, CNFPARAM_REQUIRED }, + { "file", eCmdHdlrString, CNFPARAM_REQUIRED } +}; +static struct cnfparamblk modpblk = + { CNFPARAMBLK_VERSION, + sizeof(modpdescr)/sizeof(struct cnfparamdescr), + modpdescr + }; rsRetVal lookupNew(lookup_t **ppThis, char *modname, char *dynname) @@ -56,15 +66,38 @@ lookupDestruct(lookup_t *lookup) free(lookup); } +rsRetVal +lookupProcessCnf(struct cnfobj *o) +{ + struct cnfparamvals *pvals; + uchar *cnfModName = NULL; + int typeIdx; + DEFiRet; + + pvals = nvlstGetParams(o->nvlst, &modpblk, NULL); + if(pvals == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + DBGPRINTF("lookupProcessCnf params:\n"); + cnfparamsPrint(&modpblk, pvals); + + // TODO: add code + +finalize_it: + free(cnfModName); + cnfparamvalsDestruct(pvals, &modpblk); + RETiRet; +} + void -lookupModExit(void) +lookupClassExit(void) { objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); } rsRetVal -lookupModInit(void) +lookupClassInit(void) { DEFiRet; CHKiRet(objGetObjInterface(&obj)); diff --git a/runtime/lookup.h b/runtime/lookup.h index 5eecf215..d7f38118 100644 --- a/runtime/lookup.h +++ b/runtime/lookup.h @@ -26,8 +26,9 @@ struct lookup_s { /* prototypes */ rsRetVal lookupNew(lookup_t **ppThis, char *modname, char *dynname); +rsRetVal lookupProcessCnf(struct cnfobj *o); void lookupDestruct(lookup_t *pThis); -rsRetVal lookupModInit(void); -void lookupModExit(void); +void lookupClassExit(void); +rsRetVal lookupClassInit(void); #endif /* #ifndef INCLUDED_LOOKUP_H */ diff --git a/runtime/rsconf.c b/runtime/rsconf.c index d8b81f1b..e0b7d5eb 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -2,7 +2,7 @@ * * Module begun 2011-04-19 by Rainer Gerhards * - * Copyright 2011-2012 Adiscon GmbH. + * Copyright 2011-2013 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -413,6 +413,9 @@ void cnfDoObj(struct cnfobj *o) case CNFOBJ_INPUT: inputProcessCnf(o); break; + case CNFOBJ_LOOKUP_TABLE: + lookupProcessCnf(o); + break; case CNFOBJ_TPL: if(tplProcessCnf(o) != RS_RET_OK) parser_errmsg("error processing template object"); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 047dfa9b..f9c52bb9 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -186,6 +186,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(strgenClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "rsconf"; CHKiRet(rsconfClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "lookup"; + CHKiRet(lookupClassInit(NULL)); /* dummy "classes" */ if(ppErrObj != NULL) *ppErrObj = "str"; -- cgit v1.2.3 From 1e6918857378fecf12e80dcd4fafc08afbfa9bce Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Jul 2013 11:44:40 +0200 Subject: milestone: lookup table ready for actual load (but load not yet done) --- runtime/lookup.c | 52 +++++++++++++++++++++++++++++++++++++++++++++------- runtime/lookup.h | 10 +++++++++- runtime/rsconf.c | 1 + runtime/rsconf.h | 2 ++ runtime/rsyslog.c | 3 ++- runtime/typedefs.h | 1 + 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/runtime/lookup.c b/runtime/lookup.c index 7cb27b07..f1931838 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -30,6 +30,7 @@ #include "msg.h" #include "rsconf.h" #include "dirty.h" +#include "unicode-helper.h" /* definitions for objects we access */ DEFobjStaticHelpers @@ -48,16 +49,32 @@ static struct cnfparamblk modpblk = modpdescr }; + +/* create a new lookup table object AND include it in our list of + * lookup tables. + */ rsRetVal -lookupNew(lookup_t **ppThis, char *modname, char *dynname) +lookupNew(lookup_t **ppThis) { - lookup_t *pThis; + lookup_t *pThis = NULL; DEFiRet; - CHKmalloc(pThis = calloc(1, sizeof(lookup_t))); + CHKmalloc(pThis = malloc(sizeof(lookup_t))); + pThis->name = NULL; + + if(loadConf->lu_tabs.root == NULL) { + loadConf->lu_tabs.root = pThis; + pThis->next = NULL; + } else { + pThis->next = loadConf->lu_tabs.last; + } + loadConf->lu_tabs.last = pThis; *ppThis = pThis; finalize_it: + if(iRet != RS_RET_OK) { + free(pThis); + } RETiRet; } void @@ -66,12 +83,20 @@ lookupDestruct(lookup_t *lookup) free(lookup); } +void +lookupInitCnf(lookup_tables_t *lu_tabs) +{ + lu_tabs->root = NULL; + lu_tabs->last = NULL; +} + + rsRetVal lookupProcessCnf(struct cnfobj *o) { struct cnfparamvals *pvals; - uchar *cnfModName = NULL; - int typeIdx; + lookup_t *lu; + short i; DEFiRet; pvals = nvlstGetParams(o->nvlst, &modpblk, NULL); @@ -81,10 +106,23 @@ lookupProcessCnf(struct cnfobj *o) DBGPRINTF("lookupProcessCnf params:\n"); cnfparamsPrint(&modpblk, pvals); - // TODO: add code + CHKiRet(lookupNew(&lu)); + + for(i = 0 ; i < modpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(modpblk.descr[i].name, "file")) { + CHKmalloc(lu->filename = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL)); + } else if(!strcmp(modpblk.descr[i].name, "name")) { + CHKmalloc(lu->name = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL)); + } else { + dbgprintf("lookup_table: program error, non-handled " + "param '%s'\n", modpblk.descr[i].name); + } + } + DBGPRINTF("lookup table '%s' loaded from file '%s'\n", lu->name, lu->filename); finalize_it: - free(cnfModName); cnfparamvalsDestruct(pvals, &modpblk); RETiRet; } diff --git a/runtime/lookup.h b/runtime/lookup.h index d7f38118..8dd7c23e 100644 --- a/runtime/lookup.h +++ b/runtime/lookup.h @@ -21,11 +21,19 @@ #ifndef INCLUDED_LOOKUP_H #define INCLUDED_LOOKUP_H +struct lookup_tables_s { + lookup_t *root; /* the root of the template list */ + lookup_t *last; /* points to the last element of the template list */ +}; + struct lookup_s { + uchar *name; + uchar *filename; + lookup_t *next; }; /* prototypes */ -rsRetVal lookupNew(lookup_t **ppThis, char *modname, char *dynname); +void lookupInitCnf(lookup_tables_t *lu_tabs); rsRetVal lookupProcessCnf(struct cnfobj *o); void lookupDestruct(lookup_t *pThis); void lookupClassExit(void); diff --git a/runtime/rsconf.c b/runtime/rsconf.c index e0b7d5eb..d4f8c8af 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -124,6 +124,7 @@ BEGINobjConstruct(rsconf) /* be sure to specify the object type also in END macr pThis->templates.last = NULL; pThis->templates.lastStatic = NULL; pThis->actions.nbrActions = 0; + lookupInitCnf(&pThis->lu_tabs); CHKiRet(llInit(&pThis->rulesets.llRulesets, rulesetDestructForLinkedList, rulesetKeyDestruct, strcasecmp)); /* queue params */ diff --git a/runtime/rsconf.h b/runtime/rsconf.h index 484fec8c..894c0d12 100644 --- a/runtime/rsconf.h +++ b/runtime/rsconf.h @@ -25,6 +25,7 @@ #include "linkedlist.h" #include "queue.h" +#include "lookup.h" /* --- configuration objects (the plan is to have ALL upper layers in this file) --- */ @@ -143,6 +144,7 @@ struct rsconf_s { globals_t globals; defaults_t defaults; templates_t templates; + lookup_tables_t lu_tabs; outchannels_t och; actions_t actions; rulesets_t rulesets; diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index f9c52bb9..53c7855a 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -74,6 +74,7 @@ #include "prop.h" #include "ruleset.h" #include "parser.h" +#include "lookup.h" #include "strgen.h" #include "statsobj.h" #include "atomic.h" @@ -187,7 +188,7 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) if(ppErrObj != NULL) *ppErrObj = "rsconf"; CHKiRet(rsconfClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "lookup"; - CHKiRet(lookupClassInit(NULL)); + CHKiRet(lookupClassInit()); /* dummy "classes" */ if(ppErrObj != NULL) *ppErrObj = "str"; diff --git a/runtime/typedefs.h b/runtime/typedefs.h index 616b60c9..c962d0a8 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -100,6 +100,7 @@ typedef struct outchannels_s outchannels_t; typedef struct modConfData_s modConfData_t; typedef struct instanceConf_s instanceConf_t; typedef struct ratelimit_s ratelimit_t; +typedef struct lookup_tables_s lookup_tables_t; typedef struct lookup_s lookup_t; typedef struct action_s action_t; typedef int rs_size_t; /* we do never need more than 2Gig strings, signed permits to -- cgit v1.2.3 From 9f209df82faf84b25f94db16d04e9b19b42783f8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Jul 2013 13:01:14 +0200 Subject: milestone: read in lookup table file & json parse it --- runtime/lookup.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/rsyslog.h | 1 + 2 files changed, 78 insertions(+) diff --git a/runtime/lookup.c b/runtime/lookup.c index f1931838..c581da6c 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -22,9 +22,16 @@ #include "config.h" #include #include +#include +#include +#include +#include +#include +#include #include #include "rsyslog.h" +#include "srUtils.h" #include "errmsg.h" #include "lookup.h" #include "msg.h" @@ -91,6 +98,75 @@ lookupInitCnf(lookup_tables_t *lu_tabs) } +/* note: widely-deployed json_c 0.9 does NOT support incremental + * parsing. In order to keep compatible with e.g. Ubuntu 12.04LTS, + * we read the file into one big memory buffer and parse it at once. + * While this is not very elegant, it will not pose any real issue + * for "reasonable" lookup tables (and "unreasonably" large ones + * will probably have other issues as well...). + */ +rsRetVal +lookupReadFile(lookup_t *pThis) +{ + struct json_tokener *tokener = NULL; + struct json_object *json; + int eno = errno; + char errStr[1024]; + char *iobuf = NULL; + int fd; + ssize_t nread; + struct stat sb; + enum json_tokener_error jerr; + DEFiRet; + + + if(stat((char*)pThis->filename, &sb) == -1) { + eno = errno; + errmsg.LogError(0, RS_RET_FILE_NOT_FOUND, + "lookup table file '%s' stat failed: %s", + pThis->filename, rs_strerror_r(eno, errStr, sizeof(errStr))); + ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + } + + CHKmalloc(iobuf = malloc(sb.st_size)); + + if((fd = open((const char*) pThis->filename, O_RDONLY)) == -1) { + eno = errno; + errmsg.LogError(0, RS_RET_FILE_NOT_FOUND, + "lookup table file '%s' could not be opened: %s", + pThis->filename, rs_strerror_r(eno, errStr, sizeof(errStr))); + ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + } + + tokener = json_tokener_new(); + nread = read(fd, iobuf, sb.st_size); +dbgprintf("DDDD: read buffer of %lld bytes: '%s'\n", (long long) sb.st_size, iobuf); + if(nread != (ssize_t) sb.st_size) { + eno = errno; + errmsg.LogError(0, RS_RET_READ_ERR, + "lookup table file '%s' read error: %s", + pThis->filename, rs_strerror_r(eno, errStr, sizeof(errStr))); + ABORT_FINALIZE(RS_RET_READ_ERR); + } + + json = json_tokener_parse_ex(tokener, iobuf, sb.st_size); + if(json == NULL) { + errmsg.LogError(0, RS_RET_JSON_PARSE_ERR, + "lookup table file '%s' json parsing error", + pThis->filename); + ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR); + } + + /* got json object, now populate our own in-memory structure */ + +finalize_it: + free(iobuf); + if(tokener != NULL) + json_tokener_free(tokener); + RETiRet; +} + + rsRetVal lookupProcessCnf(struct cnfobj *o) { @@ -120,6 +196,7 @@ lookupProcessCnf(struct cnfobj *o) "param '%s'\n", modpblk.descr[i].name); } } + CHKiRet(lookupReadFile(lu)); DBGPRINTF("lookup table '%s' loaded from file '%s'\n", lu->name, lu->filename); finalize_it: diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 63112627..99bf2ece 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -417,6 +417,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_QUEUE_CRY_DISK_ONLY = -2351,/**< crypto provider only supported for disk-associated queues */ RS_RET_NO_DATA = -2352,/**< file has no data; more a state than a real error */ RS_RET_RELP_AUTH_FAIL = -2353,/**< RELP peer authentication failed */ + RS_RET_READ_ERR = -2354,/**< read error occured (file i/o) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 40d205410425e5cd26961b4d59a967235ee792f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jul 2013 09:09:37 +0200 Subject: milestone: completed string lookup table load (naive version) --- runtime/lookup.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- runtime/lookup.h | 9 +++++++++ runtime/typedefs.h | 2 ++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/runtime/lookup.c b/runtime/lookup.c index c581da6c..7ee2eba3 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -98,6 +98,54 @@ lookupInitCnf(lookup_tables_t *lu_tabs) } +/* comparison function for qsort() and bsearch() string array compare + * this is for the string lookup table type + */ +static int +qs_arrcmp_strtab(const void *s1, const void *s2) +{ + return ustrcmp(((lookup_string_tab_etry_t*)s1)->key, ((lookup_string_tab_etry_t*)s2)->key); +} + +rsRetVal +lookupBuildTable(lookup_t *pThis, struct json_object *jroot) +{ + struct json_object *jversion, *jnomatch, *jtype, *jtab; + struct json_object *jrow, *jindex, *jvalue; + uint32_t i; + uint32_t maxStrSize; + DEFiRet; + + jversion = json_object_object_get(jroot, "version"); + jnomatch = json_object_object_get(jroot, "nomatch"); + jtype = json_object_object_get(jroot, "type"); + jtab = json_object_object_get(jroot, "table"); +DBGPRINTF("DDDD: version: %d, nomatch %s, type %s, table: '%s'\n", + json_object_get_int(jversion), + json_object_get_string(jnomatch), + json_object_get_string(jtype), + json_object_get_string(jtab)); + pThis->nmemb = json_object_array_length(jtab); + CHKmalloc(pThis->d.strtab = malloc(pThis->nmemb * sizeof(lookup_string_tab_etry_t))); + + maxStrSize = 0; + for(i = 0 ; i < pThis->nmemb ; ++i) { + jrow = json_object_array_get_idx(jtab, i); + jindex = json_object_object_get(jrow, "index"); + jvalue = json_object_object_get(jrow, "value"); + CHKmalloc(pThis->d.strtab[i].key = (uchar*) strdup(json_object_get_string(jindex))); + CHKmalloc(pThis->d.strtab[i].val = (uchar*) strdup(json_object_get_string(jvalue))); + maxStrSize += ustrlen(pThis->d.strtab[i].val); + } + + qsort(pThis->d.strtab, pThis->nmemb, sizeof(lookup_string_tab_etry_t), qs_arrcmp_strtab); +dbgprintf("DDDD: table loaded (max size %u):\n", maxStrSize); +for(i = 0 ; i < pThis->nmemb ; ++i) + dbgprintf("key: '%s', val: '%s'\n", pThis->d.strtab[i].key, pThis->d.strtab[i].val); +finalize_it: + RETiRet; +} + /* note: widely-deployed json_c 0.9 does NOT support incremental * parsing. In order to keep compatible with e.g. Ubuntu 12.04LTS, * we read the file into one big memory buffer and parse it at once. @@ -116,7 +164,6 @@ lookupReadFile(lookup_t *pThis) int fd; ssize_t nread; struct stat sb; - enum json_tokener_error jerr; DEFiRet; @@ -158,6 +205,7 @@ dbgprintf("DDDD: read buffer of %lld bytes: '%s'\n", (long long) sb.st_size, iob } /* got json object, now populate our own in-memory structure */ + CHKiRet(lookupBuildTable(pThis, json)); finalize_it: free(iobuf); diff --git a/runtime/lookup.h b/runtime/lookup.h index 8dd7c23e..5dd02283 100644 --- a/runtime/lookup.h +++ b/runtime/lookup.h @@ -26,9 +26,18 @@ struct lookup_tables_s { lookup_t *last; /* points to the last element of the template list */ }; +struct lookup_string_tab_etry_s { + uchar *key; + uchar *val; +}; + struct lookup_s { uchar *name; uchar *filename; + uint32_t nmemb; + union { + lookup_string_tab_etry_t *strtab; + } d; lookup_t *next; }; diff --git a/runtime/typedefs.h b/runtime/typedefs.h index c962d0a8..a929b30c 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -25,6 +25,7 @@ */ #ifndef INCLUDED_TYPEDEFS_H #define INCLUDED_TYPEDEFS_H +#include #if defined(__FreeBSD__) #include #endif @@ -100,6 +101,7 @@ typedef struct outchannels_s outchannels_t; typedef struct modConfData_s modConfData_t; typedef struct instanceConf_s instanceConf_t; typedef struct ratelimit_s ratelimit_t; +typedef struct lookup_string_tab_etry_s lookup_string_tab_etry_t; typedef struct lookup_tables_s lookup_tables_t; typedef struct lookup_s lookup_t; typedef struct action_s action_t; -- cgit v1.2.3 From 7e6cb8885e36d158c29f3b10be8d41d5884763f4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jul 2013 12:43:01 +0200 Subject: doc: copy&paste error fix --- doc/lookup_tables.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lookup_tables.html b/doc/lookup_tables.html index d72810f1..b7fcff04 100644 --- a/doc/lookup_tables.html +++ b/doc/lookup_tables.html @@ -104,7 +104,7 @@ lookup_table(name="name" file="/path/to/file" reloadOnHUP="on|off")

lookup() Function

This function is used to actually do the table lookup. Format: 

-lookup_table("name", indexvalue)
+lookup("name", indexvalue)

Parameters

    -- cgit v1.2.3 From f8f87de0a5111b02b35057005949986ab8616802 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jul 2013 16:06:57 +0200 Subject: milestone: basic lookup() functionality is present done the naive way, string indzes only --- grammar/rainerscript.c | 50 +++++++++++++++++++++++++++++++++++++++++ grammar/rainerscript.h | 3 ++- runtime/lookup.c | 60 +++++++++++++++++++++++++++++++++++++++++++------- runtime/lookup.h | 2 ++ 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index e3e7cb32..ef44a410 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -1334,6 +1334,7 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr) int bMustFree; es_str_t *estr; char *str; + char *s; uchar *resStr; int retval; struct var r[CNFFUNC_MAX_ARGS]; @@ -1471,6 +1472,20 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr) ret->d.n = 1; ret->datatype = 'N'; break; + case CNFFUNC_LOOKUP: +dbgprintf("DDDD: executing lookup\n"); + ret->datatype = 'S'; + if(func->funcdata == NULL) { + ret->d.estr = es_newStrFromCStr("TABLE-NOT-FOUND", sizeof("TABLE-NOT-FOUND")-1); + break; + } + cnfexprEval(func->expr[1], &r[1], usrptr); + str = (char*) var2CString(&r[1], &bMustFree); + s = (char*)lookupKey(func->funcdata, (uchar*)str); + ret->d.estr = es_newStrFromCStr(s, strlen(s)); + if(bMustFree) free(str); + if(r[1].datatype == 'S') es_deleteStr(r[1].d.estr); + break; default: if(Debug) { fname = es_str2cstr(func->fname, NULL); @@ -3233,6 +3248,13 @@ funcName2ID(es_str_t *fname, unsigned short nParams) return CNFFUNC_INVALID; } return CNFFUNC_PRIFILT; + } else if(!es_strbufcmp(fname, (unsigned char*)"lookup", sizeof("lookup") - 1)) { + if(nParams != 2) { + parser_errmsg("number of parameters for lookup() must be two " + "but is %d.", nParams); + return CNFFUNC_INVALID; + } + return CNFFUNC_LOOKUP; } else { return CNFFUNC_INVALID; } @@ -3297,6 +3319,31 @@ finalize_it: } +static inline rsRetVal +initFunc_lookup(struct cnffunc *func) +{ + struct funcData_prifilt *pData; + uchar *cstr = NULL; + DEFiRet; + + func->funcdata = NULL; + if(func->expr[0]->nodetype != 'S') { + parser_errmsg("table name (param 1) of lookup() must be a constant string"); + FINALIZE; + } + + cstr = (uchar*)es_str2cstr(((struct cnfstringval*) func->expr[0])->estr, NULL); + if((func->funcdata = lookupFindTable(cstr)) == NULL) { + parser_errmsg("lookup table '%s' not found", cstr); + FINALIZE; + } + +finalize_it: + free(cstr); + RETiRet; +} + + struct cnffunc * cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) { @@ -3334,6 +3381,9 @@ cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) case CNFFUNC_PRIFILT: initFunc_prifilt(func); break; + case CNFFUNC_LOOKUP: + initFunc_lookup(func); + break; default:break; } } diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h index cb8c968f..5c9a05ac 100644 --- a/grammar/rainerscript.h +++ b/grammar/rainerscript.h @@ -232,7 +232,8 @@ enum cnffuncid { CNFFUNC_RE_MATCH, CNFFUNC_RE_EXTRACT, CNFFUNC_FIELD, - CNFFUNC_PRIFILT + CNFFUNC_PRIFILT, + CNFFUNC_LOOKUP }; struct cnffunc { diff --git a/runtime/lookup.c b/runtime/lookup.c index 7ee2eba3..e83f5979 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -98,7 +98,7 @@ lookupInitCnf(lookup_tables_t *lu_tabs) } -/* comparison function for qsort() and bsearch() string array compare +/* comparison function for qsort() and string array compare * this is for the string lookup table type */ static int @@ -106,6 +106,14 @@ qs_arrcmp_strtab(const void *s1, const void *s2) { return ustrcmp(((lookup_string_tab_etry_t*)s1)->key, ((lookup_string_tab_etry_t*)s2)->key); } +/* comparison function for bsearch() and string array compare + * this is for the string lookup table type + */ +static int +bs_arrcmp_strtab(const void *s1, const void *s2) +{ + return strcmp((char*)s1, (char*)((lookup_string_tab_etry_t*)s2)->key); +} rsRetVal lookupBuildTable(lookup_t *pThis, struct json_object *jroot) @@ -120,11 +128,6 @@ lookupBuildTable(lookup_t *pThis, struct json_object *jroot) jnomatch = json_object_object_get(jroot, "nomatch"); jtype = json_object_object_get(jroot, "type"); jtab = json_object_object_get(jroot, "table"); -DBGPRINTF("DDDD: version: %d, nomatch %s, type %s, table: '%s'\n", - json_object_get_int(jversion), - json_object_get_string(jnomatch), - json_object_get_string(jtype), - json_object_get_string(jtab)); pThis->nmemb = json_object_array_length(jtab); CHKmalloc(pThis->d.strtab = malloc(pThis->nmemb * sizeof(lookup_string_tab_etry_t))); @@ -142,10 +145,48 @@ DBGPRINTF("DDDD: version: %d, nomatch %s, type %s, table: '%s'\n", dbgprintf("DDDD: table loaded (max size %u):\n", maxStrSize); for(i = 0 ; i < pThis->nmemb ; ++i) dbgprintf("key: '%s', val: '%s'\n", pThis->d.strtab[i].key, pThis->d.strtab[i].val); + finalize_it: RETiRet; } + +/* find a lookup table. This is a naive O(n) algo, but this really + * doesn't matter as it is called only a few times during config + * load. The function returns either a pointer to the requested + * table or NULL, if not found. + */ +lookup_t * +lookupFindTable(uchar *name) +{ + lookup_t *curr; + + for(curr = loadConf->lu_tabs.root ; curr != NULL ; curr = curr->next) { + if(!ustrcmp(curr->name, name)) + break; + } + return curr; +} + + +/* returns either a pointer to the value (read only!) or NULL + * if either the key could not be found or an error occured. + */ +uchar * +lookupKey(lookup_t *pThis, uchar *key) +{ + lookup_string_tab_etry_t *etry; + uchar *r; + etry = bsearch(key, pThis->d.strtab, pThis->nmemb, sizeof(lookup_string_tab_etry_t), bs_arrcmp_strtab); + if(etry == NULL) { + r = (uchar*)""; // TODO: use set default + } else { + r = etry->val; + } + return r; +} + + /* note: widely-deployed json_c 0.9 does NOT support incremental * parsing. In order to keep compatible with e.g. Ubuntu 12.04LTS, * we read the file into one big memory buffer and parse it at once. @@ -157,7 +198,7 @@ rsRetVal lookupReadFile(lookup_t *pThis) { struct json_tokener *tokener = NULL; - struct json_object *json; + struct json_object *json = NULL; int eno = errno; char errStr[1024]; char *iobuf = NULL; @@ -187,7 +228,6 @@ lookupReadFile(lookup_t *pThis) tokener = json_tokener_new(); nread = read(fd, iobuf, sb.st_size); -dbgprintf("DDDD: read buffer of %lld bytes: '%s'\n", (long long) sb.st_size, iobuf); if(nread != (ssize_t) sb.st_size) { eno = errno; errmsg.LogError(0, RS_RET_READ_ERR, @@ -203,6 +243,8 @@ dbgprintf("DDDD: read buffer of %lld bytes: '%s'\n", (long long) sb.st_size, iob pThis->filename); ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR); } + free(iobuf); /* early free to sever resources*/ + iobuf = NULL; /* make sure no double-free */ /* got json object, now populate our own in-memory structure */ CHKiRet(lookupBuildTable(pThis, json)); @@ -211,6 +253,8 @@ finalize_it: free(iobuf); if(tokener != NULL) json_tokener_free(tokener); + if(json != NULL) + json_object_put(json); RETiRet; } diff --git a/runtime/lookup.h b/runtime/lookup.h index 5dd02283..db2737a2 100644 --- a/runtime/lookup.h +++ b/runtime/lookup.h @@ -44,6 +44,8 @@ struct lookup_s { /* prototypes */ void lookupInitCnf(lookup_tables_t *lu_tabs); rsRetVal lookupProcessCnf(struct cnfobj *o); +lookup_t *lookupFindTable(uchar *name); +uchar *lookupKey(lookup_t *pThis, uchar *key); void lookupDestruct(lookup_t *pThis); void lookupClassExit(void); rsRetVal lookupClassInit(void); -- cgit v1.2.3 From 6caa8cb043885c1ea64f39d87e18e63d3c6c36ea Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jul 2013 11:51:09 +0200 Subject: add base plumbing for (later) dynamic table reload among others, we change some internal interfaces. So far, we only add the necessary locks. More work in later commits. --- grammar/rainerscript.c | 3 +-- runtime/lookup.c | 26 ++++++++++++++++++-------- runtime/lookup.h | 5 ++++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index ef44a410..53e56f05 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -1481,8 +1481,7 @@ dbgprintf("DDDD: executing lookup\n"); } cnfexprEval(func->expr[1], &r[1], usrptr); str = (char*) var2CString(&r[1], &bMustFree); - s = (char*)lookupKey(func->funcdata, (uchar*)str); - ret->d.estr = es_newStrFromCStr(s, strlen(s)); + ret->d.estr = lookupKey_estr(func->funcdata, (uchar*)str); if(bMustFree) free(str); if(r[1].datatype == 'S') es_deleteStr(r[1].d.estr); break; diff --git a/runtime/lookup.c b/runtime/lookup.c index e83f5979..b0780354 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -67,6 +67,7 @@ lookupNew(lookup_t **ppThis) DEFiRet; CHKmalloc(pThis = malloc(sizeof(lookup_t))); + pthread_rwlock_init(&pThis->rwlock, NULL); pThis->name = NULL; if(loadConf->lu_tabs.root == NULL) { @@ -85,9 +86,11 @@ finalize_it: RETiRet; } void -lookupDestruct(lookup_t *lookup) +lookupDestruct(lookup_t *pThis) { - free(lookup); + pthread_rwlock_destroy(&pThis->rwlock); + free(pThis->name); + free(pThis); } void @@ -171,19 +174,26 @@ lookupFindTable(uchar *name) /* returns either a pointer to the value (read only!) or NULL * if either the key could not be found or an error occured. + * Note that an estr_t object is returned. The caller is + * responsible for freeing it. */ -uchar * -lookupKey(lookup_t *pThis, uchar *key) +es_str_t * +lookupKey_estr(lookup_t *pThis, uchar *key) { lookup_string_tab_etry_t *etry; - uchar *r; + char *r; + es_str_t *estr; + + pthread_rwlock_rdlock(&pThis->rwlock); etry = bsearch(key, pThis->d.strtab, pThis->nmemb, sizeof(lookup_string_tab_etry_t), bs_arrcmp_strtab); if(etry == NULL) { - r = (uchar*)""; // TODO: use set default + r = ""; // TODO: use set default } else { - r = etry->val; + r = (char*)etry->val; } - return r; + estr = es_newStrFromCStr(r, strlen(r)); + pthread_rwlock_unlock(&pThis->rwlock); + return estr; } diff --git a/runtime/lookup.h b/runtime/lookup.h index db2737a2..12bd9336 100644 --- a/runtime/lookup.h +++ b/runtime/lookup.h @@ -20,6 +20,7 @@ */ #ifndef INCLUDED_LOOKUP_H #define INCLUDED_LOOKUP_H +#include struct lookup_tables_s { lookup_t *root; /* the root of the template list */ @@ -31,7 +32,9 @@ struct lookup_string_tab_etry_s { uchar *val; }; +/* a single lookup table */ struct lookup_s { + pthread_rwlock_t rwlock; /* protect us in case of dynamic reloads */ uchar *name; uchar *filename; uint32_t nmemb; @@ -45,7 +48,7 @@ struct lookup_s { void lookupInitCnf(lookup_tables_t *lu_tabs); rsRetVal lookupProcessCnf(struct cnfobj *o); lookup_t *lookupFindTable(uchar *name); -uchar *lookupKey(lookup_t *pThis, uchar *key); +es_str_t *lookupKey(lookup_t *pThis, uchar *key); void lookupDestruct(lookup_t *pThis); void lookupClassExit(void); rsRetVal lookupClassInit(void); -- cgit v1.2.3 From 32467b3bcd7cfd755117b24675a9985135179207 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jul 2013 15:07:24 +0200 Subject: reload lookup tables on HUP --- runtime/lookup.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++- runtime/lookup.h | 3 ++- tools/syslogd.c | 1 + 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/runtime/lookup.c b/runtime/lookup.c index b0780354..bc3b1a94 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -44,6 +44,9 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) +/* forward definitions */ +static rsRetVal lookupReadFile(lookup_t *pThis); + /* static data */ /* tables for interfacing with the v6 config system (as far as we need to) */ static struct cnfparamdescr modpdescr[] = { @@ -172,6 +175,53 @@ lookupFindTable(uchar *name) } +/* this reloads a lookup table. This is done while the engine is running, + * as such the function must ensure proper locking and proper order of + * operations (so that nothing can interfere). If the table cannot be loaded, + * the old table is continued to be used. + */ +static rsRetVal +lookupReload(lookup_t *pThis) +{ + uint32_t i; + lookup_t newlu; /* dummy to be able to use support functions without + affecting current settings. */ + DEFiRet; + + DBGPRINTF("reload requested for lookup table '%s'\n", pThis->name); + memset(&newlu, 0, sizeof(newlu)); + CHKmalloc(newlu.name = ustrdup(pThis->name)); + CHKmalloc(newlu.filename = ustrdup(pThis->filename)); + CHKiRet(lookupReadFile(&newlu)); + /* all went well, copy over data members */ + pthread_rwlock_wrlock(&pThis->rwlock); + for(i = 0 ; i < pThis->nmemb ; ++i) { + free(pThis->d.strtab[i].key), /* we don't care about exec order of frees */ + free(pThis->d.strtab[i].val); + } + free(pThis->d.strtab); + pThis->d.strtab = newlu.d.strtab; /* hand table AND ALL STRINGS over! */ + pthread_rwlock_unlock(&pThis->rwlock); + errmsg.LogError(0, RS_RET_OK, "lookup table '%s' reloaded from file '%s'", + pThis->name, pThis->filename); +finalize_it: + free(newlu.name); + free(newlu.filename); + RETiRet; +} + + +/* reload all lookup tables on HUP */ +void +lookupDoHUP() +{ + lookup_t *lu; + for(lu = loadConf->lu_tabs.root ; lu != NULL ; lu = lu->next) { + lookupReload(lu); + } +} + + /* returns either a pointer to the value (read only!) or NULL * if either the key could not be found or an error occured. * Note that an estr_t object is returned. The caller is @@ -204,7 +254,7 @@ lookupKey_estr(lookup_t *pThis, uchar *key) * for "reasonable" lookup tables (and "unreasonably" large ones * will probably have other issues as well...). */ -rsRetVal +static rsRetVal lookupReadFile(lookup_t *pThis) { struct json_tokener *tokener = NULL; diff --git a/runtime/lookup.h b/runtime/lookup.h index 12bd9336..c478d679 100644 --- a/runtime/lookup.h +++ b/runtime/lookup.h @@ -48,9 +48,10 @@ struct lookup_s { void lookupInitCnf(lookup_tables_t *lu_tabs); rsRetVal lookupProcessCnf(struct cnfobj *o); lookup_t *lookupFindTable(uchar *name); -es_str_t *lookupKey(lookup_t *pThis, uchar *key); +es_str_t * lookupKey_estr(lookup_t *pThis, uchar *key); void lookupDestruct(lookup_t *pThis); void lookupClassExit(void); +void lookupDoHUP(); rsRetVal lookupClassInit(void); #endif /* #ifndef INCLUDED_LOOKUP_H */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 2f0f64c3..fba60df5 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1253,6 +1253,7 @@ doHUP(void) queryLocalHostname(); /* re-read our name */ ruleset.IterateAllActions(ourConf, doHUPActions, NULL); + lookupDoHUP(); } -- cgit v1.2.3