diff options
Diffstat (limited to 'runtime')
68 files changed, 7147 insertions, 1952 deletions
diff --git a/runtime/Makefile.am b/runtime/Makefile.am index f49aac91..dea06fe0 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -17,6 +17,8 @@ librsyslog_la_SOURCES = \ module-template.h \ im-helper.h \ obj-types.h \ + sigprov.h \ + cryprov.h \ nsd.h \ glbl.h \ glbl.c \ @@ -63,16 +65,15 @@ librsyslog_la_SOURCES = \ queue.h \ ruleset.c \ ruleset.h \ - rule.c \ - rule.h \ prop.c \ prop.h \ + ratelimit.c \ + ratelimit.h \ cfsysline.c \ cfsysline.h \ sd-daemon.c \ sd-daemon.h \ \ - \ ../action.h \ ../action.c \ ../threads.c \ @@ -93,14 +94,15 @@ librsyslog_la_SOURCES = \ ../template.h # the files with ../ we need to work on - so that they either become part of the # runtime or will no longer be needed. -- rgerhards, 2008-06-13 +# if WITH_MODDIRS -librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS) $(LIBEE_CFLAGS) -I\$(top_srcdir)/tools +librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS) -I\$(top_srcdir)/tools else -librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS) $(LIBEE_CFLAGS) -I\$(top_srcdir)/tools -I\$(top_srcdir)/grammar +librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS) -I\$(top_srcdir)/tools -I\$(top_srcdir)/grammar endif #librsyslog_la_LDFLAGS = -module -avoid-version -librsyslog_la_LIBADD = $(DL_LIBS) $(RT_LIBS) $(LIBEE_LIBS) +librsyslog_la_LIBADD = $(DL_LIBS) $(RT_LIBS) # # regular expression support @@ -131,7 +133,7 @@ pkglib_LTLIBRARIES += lmnet.la lmnetstrms.la # lmnet_la_SOURCES = net.c net.h lmnet_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -lmnet_la_LDFLAGS = -module -avoid-version +lmnet_la_LDFLAGS = -module -avoid-version ../compat/compat_la-getifaddrs.lo lmnet_la_LIBADD = # network stream master class and stream factory @@ -173,6 +175,35 @@ lmnsd_gtls_la_LDFLAGS = -module -avoid-version lmnsd_gtls_la_LIBADD = $(GNUTLS_LIBS) endif +# +# support library for libgcrypt +# +if ENABLE_LIBGCRYPT + noinst_LTLIBRARIES += libgcry.la + libgcry_la_SOURCES = libgcry.c libgcry_common.c libgcry.h + libgcry_la_CPPFLAGS = $(RSRT_CFLAGS) $(LIBGCRYPT_CFLAGS) + pkglib_LTLIBRARIES += lmcry_gcry.la + lmcry_gcry_la_SOURCES = lmcry_gcry.c lmcry_gcry.h + lmcry_gcry_la_CPPFLAGS = $(RSRT_CFLAGS) $(LIBGCRYPT_CFLAGS) + lmcry_gcry_la_LDFLAGS = -module -avoid-version + lmcry_gcry_la_LIBADD = libgcry.la $(LIBGCRYPT_LIBS) +endif + + +# +# support library for guardtime +# +if ENABLE_GUARDTIME + noinst_LTLIBRARIES += librsgt.la + librsgt_la_SOURCES = librsgt.c librsgt_read.c librsgt.h + pkglib_LTLIBRARIES += lmsig_gt.la + lmsig_gt_la_SOURCES = lmsig_gt.c lmsig_gt.h + lmsig_gt_la_CPPFLAGS = $(RSRT_CFLAGS) $(GUARDTIME_CFLAGS) + lmsig_gt_la_LDFLAGS = -module -avoid-version + lmsig_gt_la_LIBADD = librsgt.la $(GUARDTIME_LIBS) +endif + + update-systemd: curl http://cgit.freedesktop.org/systemd/systemd/plain/src/libsystemd-daemon/sd-daemon.c > sd-daemon.c curl http://cgit.freedesktop.org/systemd/systemd/plain/src/systemd/sd-daemon.h > sd-daemon.h diff --git a/runtime/batch.h b/runtime/batch.h index 944889bd..2ec07670 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -34,24 +34,21 @@ * main message queue. But over time, it could potentially be useful to split the two. * rgerhad, 2009-05-12 */ -typedef enum { - BATCH_STATE_RDY = 0, /* object ready for processing */ - BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ - BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unknown */ - BATCH_STATE_COMM = 3, /* message successfully commited */ - BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */ -} batch_state_t; +#define BATCH_STATE_RDY 0 /* object ready for processing */ +#define BATCH_STATE_BAD 1 /* unrecoverable failure while processing, do NOT resubmit to same action */ +#define BATCH_STATE_SUB 2 /* message submitted for processing, outcome yet unknown */ +#define BATCH_STATE_COMM 3 /* message successfully commited */ +#define BATCH_STATE_DISC 4 /* discarded - processed OK, but do not submit to any other action */ +typedef unsigned char batch_state_t; /* an object inside a batch, including any information (state!) needed for it to "life". */ struct batch_obj_s { - obj_t *pUsrp; /* pointer to user object (most often message) */ - batch_state_t state; /* associated state */ + msg_t *pMsg; /* work variables for action processing; these are reused for each action (or block of * actions) */ - sbool bFilterOK; /* work area for filter processing (per action, reused!) */ sbool bPrevWasSuspended; /* following are caches to save allocs if not absolutely necessary */ uchar *staticActStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /**< for strings */ @@ -83,8 +80,16 @@ struct batch_s { int iDoneUpTo; /* all messages below this index have state other than RDY */ qDeqID deqID; /* ID of dequeue operation that generated this batch */ int *pbShutdownImmediate;/* end processing of this batch immediately if set to 1 */ + sbool *active; /* which messages are active for processing, NULL=all */ sbool bSingleRuleset; /* do all msgs of this batch use a single ruleset? */ batch_obj_t *pElem; /* batch elements */ + batch_state_t *eltState;/* state (array!) for individual objects. + NOTE: we have moved this out of batch_obj_t because we + get a *much* better cache hit ratio this way. So do not + move it back into this structure! Note that this is really + a HUGE saving, even if it doesn't look so (both profiler + data as well as practical tests indicate that!). + */ }; @@ -97,13 +102,13 @@ batchSetSingleRuleset(batch_t *pBatch, sbool val) { /* get the batches ruleset (if we have a single ruleset) */ static inline ruleset_t* batchGetRuleset(batch_t *pBatch) { - return (pBatch->nElem > 0) ? ((msg_t*) pBatch->pElem[0].pUsrp)->pRuleset : NULL; + return (pBatch->nElem > 0) ? pBatch->pElem[0].pMsg->pRuleset : NULL; } /* get the ruleset of a specifc element of the batch (index not verified!) */ static inline ruleset_t* batchElemGetRuleset(batch_t *pBatch, int i) { - return ((msg_t*) pBatch->pElem[i].pUsrp)->pRuleset; + return pBatch->pElem[i].pMsg->pRuleset; } /* get number of msgs for this batch */ @@ -119,8 +124,8 @@ batchNumMsgs(batch_t *pBatch) { */ static inline void batchSetElemState(batch_t *pBatch, int i, batch_state_t newState) { - if(pBatch->pElem[i].state != BATCH_STATE_DISC) - pBatch->pElem[i].state = newState; + if(pBatch->eltState[i] != BATCH_STATE_DISC) + pBatch->eltState[i] = newState; } @@ -129,23 +134,8 @@ batchSetElemState(batch_t *pBatch, int i, batch_state_t newState) { */ static inline int batchIsValidElem(batch_t *pBatch, int i) { - return(pBatch->pElem[i].bFilterOK && pBatch->pElem[i].state != BATCH_STATE_DISC); -} - - -/* copy one batch element to another. - * This creates a complete duplicate in those cases where - * it is needed. Use duplication only when absolutely necessary! - * Note that all working fields are reset to zeros. If that were - * not done, we would have potential problems with invalid - * or double pointer frees. - * rgerhards, 2010-06-10 - */ -static inline void -batchCopyElem(batch_obj_t *pDest, batch_obj_t *pSrc) { - memset(pDest, 0, sizeof(batch_obj_t)); - pDest->pUsrp = pSrc->pUsrp; - pDest->state = pSrc->state; + return( (pBatch->eltState[i] != BATCH_STATE_DISC) + && (pBatch->active == NULL || pBatch->active[i])); } @@ -166,6 +156,7 @@ batchFree(batch_t *pBatch) { } } free(pBatch->pElem); + free(pBatch->eltState); } @@ -179,6 +170,7 @@ batchInit(batch_t *pBatch, int maxElem) { pBatch->iDoneUpTo = 0; pBatch->maxElem = maxElem; CHKmalloc(pBatch->pElem = calloc((size_t)maxElem, sizeof(batch_obj_t))); + CHKmalloc(pBatch->eltState = calloc((size_t)maxElem, sizeof(batch_state_t))); // TODO: replace calloc by inidividual writes? finalize_it: RETiRet; diff --git a/runtime/conf.c b/runtime/conf.c index 488d1b86..c3c7e447 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -61,17 +61,16 @@ #include "srUtils.h" #include "errmsg.h" #include "net.h" -#include "rule.h" #include "ruleset.h" #include "rsconf.h" #include "unicode-helper.h" +#include "rainerscript.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN #endif /* forward definitions */ -//static rsRetVal cfline(rsconf_t *conf, uchar *line, rule_t **pfCurr); /* static data */ @@ -79,7 +78,6 @@ DEFobjStaticHelpers DEFobjCurrIf(module) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) -DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) int bConfStrictScoping = 0; /* force strict scoping during config processing? */ @@ -131,6 +129,23 @@ finalize_it: } +/* remove leading spaces from name; this "fixes" some anomalies in + * getSubString(), but I was not brave enough to fix the former as + * it has many other callers... -- rgerhards, 2013-05-27 + */ +static inline void +ltrim(char *src) +{ + char *dst = src; + while(isspace(*src)) + ++src; /*SKIP*/; + if(dst != src) { + while(*src != '\0') + *dst++ = *src++; + *dst = '\0'; + } +} + /* parse and interpret a $-config line that starts with * a name (this is common code). It is parsed to the name * and then the proper sub-function is called to handle @@ -157,6 +172,7 @@ doNameLine(uchar **pp, void* pVal) errmsg.LogError(0, RS_RET_NOT_FOUND, "Invalid config line: could not extract name - line ignored"); ABORT_FINALIZE(RS_RET_NOT_FOUND); } + ltrim(szName); if(*p == ',') ++p; /* comma was eaten */ @@ -326,14 +342,9 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int } -/* Helper to cfline(). This function takes the filter part of a traditional, PRI - * based line and decodes the PRIs given in the selector line. It processed the - * line up to the beginning of the action part. A pointer to that beginnig is - * passed back to the caller. - * rgerhards 2005-09-15 - */ +/* Decode a traditional PRI filter */ /* GPLv3 - stems back to sysklogd */ -rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) +rsRetVal DecodePRIFilter(uchar *pline, uchar pmask[]) { uchar *p; register uchar *q; @@ -347,22 +358,15 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) DEFiRet; ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ISOBJ_TYPE_assert(pRule, rule); - dbgprintf(" - traditional PRI filter '%s'\n", *pline); - errno = 0; /* keep strerror_r() stuff out of logerror messages */ + dbgprintf("Decoding traditional PRI filter '%s'\n", pline); - pRule->f_filter_type = FILTER_PRI; - /* Note: file structure is pre-initialized to zero because it was - * created with calloc()! - */ for (i = 0; i <= LOG_NFACILITIES; i++) { - pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; + pmask[i] = TABLE_NOPRI; } /* scan through the list of selectors */ - for (p = *pline; *p && *p != '\t' && *p != ' ';) { + for (p = pline; *p && *p != '\t' && *p != ' ';) { /* find the end of this facility name list */ for (q = p; *q && *q != '\t' && *q++ != '.'; ) continue; @@ -411,28 +415,28 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) for (i = 0; i <= LOG_NFACILITIES; i++) { if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pmask[i] = TABLE_ALLPRI; else - pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; + pmask[i] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i] &= ~(1<<pri); + pmask[i] &= ~(1<<pri); else - pRule->f_filterData.f_pmask[i] |= (1<<pri); + pmask[i] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; + pmask[i] = TABLE_NOPRI; else - pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pmask[i] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i] &= ~(1<<i2); + pmask[i] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i] |= (1<<i2); + pmask[i] |= (1<<i2); } } } @@ -447,27 +451,27 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pmask[i >> 3] = TABLE_ALLPRI; else - pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pmask[i >> 3] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<pri); + pmask[i >> 3] &= ~(1<<pri); else - pRule->f_filterData.f_pmask[i >> 3] |= (1<<pri); + pmask[i >> 3] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pmask[i >> 3] = TABLE_NOPRI; else - pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pmask[i >> 3] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<i2); + pmask[i >> 3] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i >> 3] |= (1<<i2); + pmask[i >> 3] |= (1<<i2); } } } @@ -478,11 +482,6 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) p = q; } - /* skip to action part */ - while (*p == '\t' || *p == ' ') - p++; - - *pline = p; RETiRet; } @@ -492,7 +491,7 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) * of the action part. A pointer to that beginnig is passed back to the caller. * rgerhards 2005-09-15 */ -rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) +rsRetVal DecodePropFilter(uchar *pline, struct cnfstmt *stmt) { rsParsObj *pPars; cstr_t *pCSCompOp; @@ -501,16 +500,11 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) int iOffset; /* for compare operations */ ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ASSERT(f != NULL); - dbgprintf(" - property-based filter '%s'\n", *pline); - errno = 0; /* keep strerror_r() stuff out of logerror messages */ - - f->f_filter_type = FILTER_PROP; + dbgprintf("Decoding property-based filter '%s'\n", pline); /* create parser object starting with line string without leading colon */ - if((iRet = rsParsConstructFromSz(&pPars, (*pline)+1)) != RS_RET_OK) { + if((iRet = rsParsConstructFromSz(&pPars, pline+1)) != RS_RET_OK) { errmsg.LogError(0, iRet, "Error %d constructing parser object - ignoring selector", iRet); return(iRet); } @@ -522,15 +516,15 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) rsParsDestruct(pPars); return(iRet); } - iRet = propNameToID(pCSPropName, &f->f_filterData.prop.propID); + iRet = propNameToID(pCSPropName, &stmt->d.s_propfilt.propID); if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); rsParsDestruct(pPars); return(iRet); } - if(f->f_filterData.prop.propID == PROP_CEE) { + if(stmt->d.s_propfilt.propID == PROP_CEE) { /* in CEE case, we need to preserve the actual property name */ - if((f->f_filterData.prop.propName = + if((stmt->d.s_propfilt.propName = es_newStrFromBuf((char*)cstrGetSzStrNoNULL(pCSPropName)+2, cstrLen(pCSPropName)-2)) == NULL) { cstrDestruct(&pCSPropName); return(RS_RET_ERR); @@ -553,38 +547,38 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) */ if(rsCStrLen(pCSCompOp) > 0) { if(*rsCStrGetBufBeg(pCSCompOp) == '!') { - f->f_filterData.prop.isNegated = 1; + stmt->d.s_propfilt.isNegated = 1; iOffset = 1; /* ignore '!' */ } else { - f->f_filterData.prop.isNegated = 0; + stmt->d.s_propfilt.isNegated = 0; iOffset = 0; } } else { - f->f_filterData.prop.isNegated = 0; + stmt->d.s_propfilt.isNegated = 0; iOffset = 0; } if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "contains", 8)) { - f->f_filterData.prop.operation = FIOP_CONTAINS; + stmt->d.s_propfilt.operation = FIOP_CONTAINS; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "isequal", 7)) { - f->f_filterData.prop.operation = FIOP_ISEQUAL; + stmt->d.s_propfilt.operation = FIOP_ISEQUAL; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "isempty", 7)) { - f->f_filterData.prop.operation = FIOP_ISEMPTY; + stmt->d.s_propfilt.operation = FIOP_ISEMPTY; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "startswith", 10)) { - f->f_filterData.prop.operation = FIOP_STARTSWITH; + stmt->d.s_propfilt.operation = FIOP_STARTSWITH; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "regex", 5)) { - f->f_filterData.prop.operation = FIOP_REGEX; + stmt->d.s_propfilt.operation = FIOP_REGEX; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "ereregex", 8)) { - f->f_filterData.prop.operation = FIOP_EREREGEX; + stmt->d.s_propfilt.operation = FIOP_EREREGEX; } else { errmsg.LogError(0, NO_ERRCODE, "error: invalid compare operation '%s' - ignoring selector", (char*) rsCStrGetSzStrNoNULL(pCSCompOp)); } rsCStrDestruct(&pCSCompOp); /* no longer needed */ - if(f->f_filterData.prop.operation != FIOP_ISEMPTY) { + if(stmt->d.s_propfilt.operation != FIOP_ISEMPTY) { /* read compare value */ - iRet = parsQuotedCStr(pPars, &f->f_filterData.prop.pCSCompValue); + iRet = parsQuotedCStr(pPars, &stmt->d.s_propfilt.pCSCompValue); if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "error %d compare value property - ignoring selector", iRet); rsParsDestruct(pPars); @@ -592,114 +586,10 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) } } - /* skip to action part */ - if((iRet = parsSkipWhitespace(pPars)) != RS_RET_OK) { - errmsg.LogError(0, iRet, "error %d skipping to action part - ignoring selector", iRet); - rsParsDestruct(pPars); - return(iRet); - } - - /* cleanup */ - *pline = *pline + rsParsGetParsePointer(pPars) + 1; - /* we are adding one for the skipped initial ":" */ - return rsParsDestruct(pPars); } -/* - * Helper to cfline(). This function interprets a BSD host selector line - * from the config file ("+/-hostname"). It stores it for further reference. - * rgerhards 2005-10-19 - */ -rsRetVal cflineProcessHostSelector(uchar **pline) -{ - DEFiRet; - - ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ASSERT(**pline == '-' || **pline == '+'); - - dbgprintf(" - host selector line\n"); - - /* check include/exclude setting */ - if(**pline == '+') { - eDfltHostnameCmpMode = HN_COMP_MATCH; - } else { /* we do not check for '-', it must be, else we wouldn't be here */ - eDfltHostnameCmpMode = HN_COMP_NOMATCH; - } - (*pline)++; /* eat + or - */ - - /* the below is somewhat of a quick hack, but it is efficient (this is - * why it is in here. "+*" resets the tag selector with BSD syslog. We mimic - * this, too. As it is easy to check that condition, we do not fire up a - * parser process, just make sure we do not address beyond our space. - * Order of conditions in the if-statement is vital! rgerhards 2005-10-18 - */ - if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { - dbgprintf("resetting BSD-like hostname filter\n"); - eDfltHostnameCmpMode = HN_NO_COMP; - if(pDfltHostnameCmp != NULL) { - CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, NULL)); - } - } else { - dbgprintf("setting BSD-like hostname filter to '%s'\n", *pline); - if(pDfltHostnameCmp == NULL) { - /* create string for parser */ - CHKiRet(rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)); - } else { /* string objects exists, just update... */ - CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, *pline)); - } - } - -finalize_it: - RETiRet; -} - - -/* - * Helper to cfline(). This function interprets a BSD tag selector line - * from the config file ("!tagname"). It stores it for further reference. - * rgerhards 2005-10-18 - */ -rsRetVal cflineProcessTagSelector(uchar **pline) -{ - DEFiRet; - - ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ASSERT(**pline == '!'); - - dbgprintf(" - programname selector line\n"); - - (*pline)++; /* eat '!' */ - - /* the below is somewhat of a quick hack, but it is efficient (this is - * why it is in here. "!*" resets the tag selector with BSD syslog. We mimic - * this, too. As it is easy to check that condition, we do not fire up a - * parser process, just make sure we do not address beyond our space. - * Order of conditions in the if-statement is vital! rgerhards 2005-10-18 - */ - if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { - dbgprintf("resetting programname filter\n"); - if(pDfltProgNameCmp != NULL) { - rsCStrDestruct(&pDfltProgNameCmp); - } - } else { - dbgprintf("setting programname filter to '%s'\n", *pline); - if(pDfltProgNameCmp == NULL) { - /* create string for parser */ - CHKiRet(rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)); - } else { /* string objects exists, just update... */ - CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, *pline)); - } - } - -finalize_it: - RETiRet; -} - - /* process the action part of a selector line * rgerhards, 2007-08-01 */ @@ -735,13 +625,8 @@ rsRetVal cflineDoAction(rsconf_t *conf, uchar **p, action_t **ppAction) if(iRet == RS_RET_OK || iRet == RS_RET_SUSPENDED) { if((iRet = addAction(&pAction, pMod, pModData, pOMSR, NULL, NULL, (iRet == RS_RET_SUSPENDED)? 1 : 0)) == RS_RET_OK) { - /* now check if the module is compatible with select features */ - if(pMod->isCompatibleWithFeature(sFEATURERepeatedMsgReduction) == RS_RET_OK) - pAction->f_ReduceRepeated = loadConf->globals.bReduceRepeatMsgs; - else { - dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); - pAction->f_ReduceRepeated = 0; - } + /* here check if the module is compatible with select features + * (currently, we have no such features!) */ pAction->eState = ACT_STATE_RDY; /* action is enabled */ conf->actions.nbrActions++; /* one more active action! */ } @@ -831,7 +716,6 @@ CODESTARTObjClassExit(conf) objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); - objRelease(rule, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); ENDObjClassExit(conf) @@ -845,7 +729,6 @@ BEGINAbstractObjClassInit(conf, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANG CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* TODO: make this dependcy go away! */ - CHKiRet(objUse(rule, CORE_COMPONENT)); CHKiRet(objUse(ruleset, CORE_COMPONENT)); /* These commands will NOT be supported -- the new v6.3 config system provides diff --git a/runtime/conf.h b/runtime/conf.h index 018d9111..a1bb51ad 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -62,11 +62,8 @@ PROTOTYPEObj(conf); rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); -/* more dirt to cover the new config interface (will go away...) */ -rsRetVal cflineProcessTagSelector(uchar **pline); -rsRetVal cflineProcessHostSelector(uchar **pline); -rsRetVal cflineProcessTradPRIFilter(uchar **pline, rule_t *pRule); -rsRetVal cflineProcessPropFilter(uchar **pline, rule_t *f); +rsRetVal DecodePRIFilter(uchar *pline, uchar pmask[]); +rsRetVal DecodePropFilter(uchar *pline, struct cnfstmt *stmt); rsRetVal cflineDoAction(rsconf_t *conf, uchar **p, action_t **ppAction); extern EHostnameCmpMode eDfltHostnameCmpMode; extern cstr_t *pDfltHostnameCmp; diff --git a/runtime/cryprov.h b/runtime/cryprov.h new file mode 100644 index 00000000..8496b745 --- /dev/null +++ b/runtime/cryprov.h @@ -0,0 +1,39 @@ +/* The interface definition for (file) crypto providers. + * + * This is just an abstract driver interface, which needs to be + * implemented by concrete classes. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_CRYPROV_H +#define INCLUDED_CRYPROV_H + +#include <gcrypt.h> + +/* interface */ +BEGINinterface(cryprov) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(void *ppThis); + rsRetVal (*SetCnfParam)(void *ppThis, struct nvlst *lst); + rsRetVal (*Destruct)(void *ppThis); + rsRetVal (*OnFileOpen)(void *pThis, uchar *fn, void *pFileInstData); + rsRetVal (*Encrypt)(void *pFileInstData, uchar *buf, size_t *lenBuf); + rsRetVal (*OnFileClose)(void *pFileInstData, off64_t offsLogfile); +ENDinterface(cryprov) +#define cryprovCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#endif /* #ifndef INCLUDED_CRYPROV_H */ diff --git a/runtime/datetime.c b/runtime/datetime.c index 0b9b1ae2..841ff625 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -61,8 +61,10 @@ timeval2syslogTime(struct timeval *tp, struct syslogTime *t) struct tm *tm; struct tm tmBuf; long lBias; + time_t secs; - tm = localtime_r((time_t*) &(tp->tv_sec), &tmBuf); + secs = tp->tv_sec; + tm = localtime_r(&secs, &tmBuf); t->year = tm->tm_year + 1900; t->month = tm->tm_mon + 1; @@ -180,12 +182,13 @@ getTime(time_t *ttSeconds) * the method always returns zero. * \retval The number parsed. */ -static int srSLMGParseInt32(uchar** ppsz, int *pLenStr) +static inline int +srSLMGParseInt32(uchar** ppsz, int *pLenStr) { register int i; i = 0; - while(*pLenStr > 0 && isdigit((int) **ppsz)) { + while(*pLenStr > 0 && **ppsz >= '0' && **ppsz <= '9') { i = i * 10 + **ppsz - '0'; ++(*ppsz); --(*pLenStr); @@ -902,6 +905,11 @@ time_t syslogTime2time_t(struct syslogTime *ts) case 12: MonthInDays = 334; //until 01 of December break; + default: /* this cannot happen (and would be a program error) + * but we need the code to keep the compiler silent. + */ + MonthInDays = 0; /* any value fits ;) */ + break; } diff --git a/runtime/debug.c b/runtime/debug.c index 307a8bb8..68474989 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -44,6 +44,9 @@ #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> +#ifdef HAVE_SYS_SYSCALL_H +# include <sys/syscall.h> +#endif #if _POSIX_TIMERS <= 0 #include <sys/time.h> #endif @@ -66,6 +69,7 @@ static int bPrintMutexAction = 0; /* shall mutex calls be printed to the debug l static int bPrintTime = 1; /* print a timestamp together with debug message */ static int bPrintAllDebugOnExit = 0; static int bAbortTrace = 1; /* print a trace after SIGABRT or SIGSEGV */ +static int bOutputTidToStderr = 0;/* output TID to stderr on thread creation */ static char *pszAltDbgFileName = NULL; /* if set, debug output is *also* sent to here */ static int altdbg = -1; /* and the handle for alternate debug output */ int stddbg = 1; /* the handle for regular debug output, set to stdout if not forking, -1 otherwise */ @@ -293,6 +297,21 @@ static inline void dbgFuncDBRemoveMutexLock(dbgFuncDB_t *pFuncDB, pthread_mutex_ /* ------------------------- END FuncDB utility functions ------------------------- */ +/* output the current thread ID to "relevant" places + * (what "relevant" means is determinded by various ways) + */ +void +dbgOutputTID(char* name) +{ +# if defined(HAVE_SYSCALL) && defined(HAVE_SYS_gettid) + if(bOutputTidToStderr) + fprintf(stderr, "thread tid %u, name '%s'\n", + (unsigned)syscall(SYS_gettid), name); + DBGPRINTF("thread created, tid %u, name '%s'\n", + (unsigned)syscall(SYS_gettid), name); +# endif +} + /* ########################################################################### * IMPORTANT NOTE * Mutex instrumentation reduces the code's concurrency and thus affects its @@ -902,8 +921,12 @@ do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg) lenCopy = lenMsg; memcpy(pszWriteBuf + offsWriteBuf, pszMsg, lenCopy); offsWriteBuf += lenCopy; - if(stddbg != -1) write(stddbg, pszWriteBuf, offsWriteBuf); - if(altdbg != -1) write(altdbg, pszWriteBuf, offsWriteBuf); + /* the write is included in an "if" just to silence compiler + * warnings. Here, we really don't care if the write fails, we + * have no good response to that in any case... -- rgerhards, 2012-11-28 + */ + if(stddbg != -1) if(write(stddbg, pszWriteBuf, offsWriteBuf)){}; + if(altdbg != -1) if(write(altdbg, pszWriteBuf, offsWriteBuf)){}; bWasNL = (pszMsg[lenMsg - 1] == '\n') ? 1 : 0; } @@ -1292,6 +1315,15 @@ dbgmalloc(size_t size) } +/* report fd used for debug log. This is needed in case of + * auto-backgrounding, where the debug log shall not be closed. + */ +int +dbgGetDbglogFd(void) +{ + return altdbg; +} + /* read in the runtime options * rgerhards, 2008-02-28 */ @@ -1321,6 +1353,7 @@ dbgGetRuntimeOptions(void) "PrintAllDebugInfoOnExit (not yet implemented)\n" "NoLogTimestamp\n" "Nostdoout\n" + "OutputTidToStderr\n" "filetrace=file (may be provided multiple times)\n" "DebugOnDemand - enables debugging on USR1, but does not turn on output\n" "\nSee debug.html in your doc set or http://www.rsyslog.com for details\n"); @@ -1354,6 +1387,8 @@ dbgGetRuntimeOptions(void) stddbg = -1; } else if(!strcasecmp((char*)optname, "noaborttrace")) { bAbortTrace = 0; + } else if(!strcasecmp((char*)optname, "outputtidtostderr")) { + bOutputTidToStderr = 1; } else if(!strcasecmp((char*)optname, "filetrace")) { if(*optval == '\0') { fprintf(stderr, "rsyslogd " VERSION " error: logfile debug option requires filename, " @@ -1372,10 +1407,30 @@ dbgGetRuntimeOptions(void) } +void +dbgSetDebugLevel(int level) +{ + Debug = level; + debugging_on = (level == DEBUG_FULL) ? 1 : 0; +} + +void +dbgSetDebugFile(uchar *fn) +{ + if(altdbg != -1) { + dbgprintf("switching to debug file %s\n", fn); + close(altdbg); + } + if((altdbg = open((char*)fn, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY|O_CLOEXEC, S_IRUSR|S_IWUSR)) == -1) { + fprintf(stderr, "alternate debug file could not be opened, ignoring. Error: %s\n", strerror(errno)); + } +} + /* end support system to set debug options at runtime */ rsRetVal dbgClassInit(void) { + pthread_mutexattr_t mutAttr; rsRetVal iRet; /* do not use DEFiRet, as this makes calls into the debug system! */ struct sigaction sigAct; @@ -1383,14 +1438,16 @@ rsRetVal dbgClassInit(void) (void) pthread_key_create(&keyCallStack, dbgCallStackDestruct); /* MUST be the first action done! */ - /* we initialize all Mutexes with code, as some platforms seem to have - * bugs in the static initializer macros. So better be on the safe side... - * rgerhards, 2008-03-06 + /* the mutexes must be recursive, because it may be called from within + * signal handlers, which can lead to a hang if the signal interrupted dbgprintf + * (yes, we have really seen that situation in practice!). -- rgerhards, 2013-05-17 */ - pthread_mutex_init(&mutFuncDBList, NULL); - pthread_mutex_init(&mutMutLog, NULL); - pthread_mutex_init(&mutCallStack, NULL); - pthread_mutex_init(&mutdbgprint, NULL); + pthread_mutexattr_init(&mutAttr); + pthread_mutexattr_settype(&mutAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&mutFuncDBList, &mutAttr); + pthread_mutex_init(&mutMutLog, &mutAttr); + pthread_mutex_init(&mutCallStack, &mutAttr); + pthread_mutex_init(&mutdbgprint, &mutAttr); /* while we try not to use any of the real rsyslog code (to avoid infinite loops), we * need to have the ability to query object names. Thus, we need to obtain a pointer to diff --git a/runtime/debug.h b/runtime/debug.h index 5bd26bd8..f3226098 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -89,6 +89,8 @@ typedef struct dbgCallStack_s { /* prototypes */ rsRetVal dbgClassInit(void); rsRetVal dbgClassExit(void); +void dbgSetDebugFile(uchar *fn); +void dbgSetDebugLevel(int level); void sigsegvHdlr(int signum); void dbgoprint(obj_t *pObj, char *fmt, ...) __attribute__((format(printf, 2, 3))); void dbgprintf(char *fmt, ...) __attribute__((format(printf, 1, 2))); @@ -104,6 +106,8 @@ void dbgSetExecLocation(int iStackPtr, int line); void dbgSetThrdName(uchar *pszName); void dbgPrintAllDebugInfo(void); void *dbgmalloc(size_t size); +void dbgOutputTID(char* name); +int dbgGetDbglogFd(void); /* macros */ #ifdef DEBUGLESS diff --git a/runtime/dnscache.c b/runtime/dnscache.c index 32d6e425..2096aa36 100644 --- a/runtime/dnscache.c +++ b/runtime/dnscache.c @@ -7,7 +7,7 @@ * In any case, even the initial implementaton is far faster than what we had * before. -- rgerhards, 2011-06-06 * - * Copyright 2011 by Rainer Gerhards and Adiscon GmbH. + * Copyright 2011-2013 by Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -33,6 +33,7 @@ #include <signal.h> #include <netdb.h> #include <unistd.h> +#include <ctype.h> #include "syslogd-types.h" #include "glbl.h" @@ -40,46 +41,99 @@ #include "obj.h" #include "unicode-helper.h" #include "net.h" +#include "hashtable.h" +#include "prop.h" +#include "dnscache.h" -/* in this initial implementation, we use a simple, non-optimized at all - * linear list. - */ /* module data structures */ struct dnscache_entry_s { struct sockaddr_storage addr; - uchar *pszHostFQDN; - uchar *ip; + prop_t *fqdn; + prop_t *fqdnLowerCase; + prop_t *localName; /* only local name, without domain part (if configured so) */ + prop_t *ip; struct dnscache_entry_s *next; unsigned nUsed; }; typedef struct dnscache_entry_s dnscache_entry_t; struct dnscache_s { pthread_rwlock_t rwlock; - dnscache_entry_t *root; + struct hashtable *ht; unsigned nEntries; }; typedef struct dnscache_s dnscache_t; -#define MAX_CACHE_ENTRIES 1000 /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) +DEFobjCurrIf(prop) static dnscache_t dnsCache; +static prop_t *staticErrValue; + + +/* Our hash function. + * TODO: check how well it performs on socket addresses! + */ +unsigned int +hash_from_key_fn(void *k) +{ + int len; + uchar *rkey = (uchar*) k; /* we treat this as opaque bytes */ + unsigned hashval = 1; + len = SALEN((struct sockaddr*)k); + while(len--) + hashval = hashval * 33 + *rkey++; + + return hashval; +} + +static int +key_equals_fn(void *key1, void *key2) +{ + return (SALEN((struct sockaddr*)key1) == SALEN((struct sockaddr*) key2) + && !memcmp(key1, key2, SALEN((struct sockaddr*) key1))); +} + +/* destruct a cache entry. + * Precondition: entry must already be unlinked from list + */ +static void +entryDestruct(dnscache_entry_t *etry) +{ + if(etry->fqdn != NULL) + prop.Destruct(&etry->fqdn); + if(etry->fqdnLowerCase != NULL) + prop.Destruct(&etry->fqdnLowerCase); + if(etry->localName != NULL) + prop.Destruct(&etry->localName); + if(etry->ip != NULL) + prop.Destruct(&etry->ip); + free(etry); +} /* init function (must be called once) */ rsRetVal dnscacheInit(void) { DEFiRet; - dnsCache.root = NULL; + if((dnsCache.ht = create_hashtable(100, hash_from_key_fn, key_equals_fn, + (void(*)(void*))entryDestruct)) == NULL) { + DBGPRINTF("dnscache: error creating hash table!\n"); + ABORT_FINALIZE(RS_RET_ERR); // TODO: make this degrade, but run! + } dnsCache.nEntries = 0; pthread_rwlock_init(&dnsCache.rwlock, NULL); CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */ CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); + + prop.Construct(&staticErrValue); + prop.SetString(staticErrValue, (uchar*)"???", 3); + prop.ConstructFinalize(staticErrValue); finalize_it: RETiRet; } @@ -89,38 +143,20 @@ rsRetVal dnscacheDeinit(void) { DEFiRet; - //TODO: free cache elements dnsCache.root = NULL; + prop.Destruct(&staticErrValue); + hashtable_destroy(dnsCache.ht, 1); /* 1 => free all values automatically */ pthread_rwlock_destroy(&dnsCache.rwlock); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); RETiRet; } -/* destruct a cache entry. - * Precondition: entry must already be unlinked from list - */ -static inline void -entryDestruct(dnscache_entry_t *etry) -{ - free(etry->pszHostFQDN); - free(etry->ip); - free(etry); -} - - static inline dnscache_entry_t* findEntry(struct sockaddr_storage *addr) { - dnscache_entry_t *etry; - for(etry = dnsCache.root ; etry != NULL ; etry = etry->next) { - if(SALEN((struct sockaddr*)addr) == SALEN((struct sockaddr*) &etry->addr) - && !memcmp(addr, &etry->addr, SALEN((struct sockaddr*) addr))) - break; /* in this case, we found our entry */ - } - if(etry != NULL) - ++etry->nUsed; /* this is *not* atomic, but we can live with an occasional loss! */ - return etry; + return((dnscache_entry_t*) hashtable_search(dnsCache.ht, addr)); } @@ -144,6 +180,73 @@ mygetnameinfo(const struct sockaddr *sa, socklen_t salen, } +/* get only the local part of the hostname and set it in cache entry */ +static inline void +setLocalHostName(dnscache_entry_t *etry) +{ + uchar *fqdnLower; + uchar *p; + int count; + int i; + uchar hostbuf[NI_MAXHOST]; + + if(glbl.GetPreserveFQDN()) { + prop.AddRef(etry->fqdnLowerCase); + etry->localName = etry->fqdnLowerCase; + goto done; + } + + /* strip domain, if configured for this entry */ + fqdnLower = propGetSzStr(etry->fqdnLowerCase); + p = (uchar*)strchr((char*)fqdnLower, '.'); /* find start of domain name "machine.example.com" */ + if(p == NULL) { /* do we have a domain part? */ + prop.AddRef(etry->fqdnLowerCase); /* no! */ + etry->localName = etry->fqdnLowerCase; + goto done; + } + + i = p - fqdnLower; /* length of hostname */ + memcpy(hostbuf, fqdnLower, i); + /* now check if we belong to any of the domain names that were specified + * in the -s command line option. If so, remove and we are done. + */ + if(glbl.GetStripDomains() != NULL) { + count=0; + while(glbl.GetStripDomains()[count]) { + if(strcmp((char*)(p + 1), glbl.GetStripDomains()[count]) == 0) { + prop.CreateStringProp(&etry->localName, hostbuf, i); + goto done; + } + count++; + } + } + /* if we reach this point, we have not found any domain we should strip. Now + * we try and see if the host itself is listed in the -l command line option + * and so should be stripped also. If so, we do it and return. Please note that + * -l list FQDNs, not just the hostname part. If it did just list the hostname, the + * door would be wide-open for all kinds of mixing up of hosts. Because of this, + * you'll see comparison against the full string (pszHostFQDN) below. + */ + if(glbl.GetLocalHosts() != NULL) { + count=0; + while(glbl.GetLocalHosts()[count]) { + if(!strcmp((char*)fqdnLower, (char*)glbl.GetLocalHosts()[count])) { + prop.CreateStringProp(&etry->localName, hostbuf, i); + goto done; + } + count++; + } + } + + /* at this point, we have not found anything, so we again use the + * already-created complete full name property. + */ + prop.AddRef(etry->fqdnLowerCase); + etry->localName = etry->fqdnLowerCase; +done: return; +} + + /* resolve an address. * * Please see http://www.hmug.org/man/3/getnameinfo.php (under Caveats) @@ -154,18 +257,19 @@ mygetnameinfo(const struct sockaddr *sa, socklen_t salen, * message should be processed (1) or discarded (0). */ static rsRetVal -resolveAddr(struct sockaddr_storage *addr, uchar *pszHostFQDN, uchar *ip) +resolveAddr(struct sockaddr_storage *addr, dnscache_entry_t *etry) { DEFiRet; int error; sigset_t omask, nmask; struct addrinfo hints, *res; + char szIP[80]; /* large enough for IPv6 */ + char fqdnBuf[NI_MAXHOST]; + rs_size_t fqdnLen; + rs_size_t i; - assert(addr != NULL); - assert(pszHostFQDN != NULL); - error = mygetnameinfo((struct sockaddr *)addr, SALEN((struct sockaddr *)addr), - (char*) ip, NI_MAXHOST, NULL, 0, NI_NUMERICHOST); + (char*) szIP, sizeof(szIP), NULL, 0, NI_NUMERICHOST); if(error) { dbgprintf("Malformed from address %s\n", gai_strerror(error)); ABORT_FINALIZE(RS_RET_INVALID_SOURCE); @@ -177,9 +281,8 @@ resolveAddr(struct sockaddr_storage *addr, uchar *pszHostFQDN, uchar *ip) pthread_sigmask(SIG_BLOCK, &nmask, &omask); error = mygetnameinfo((struct sockaddr *)addr, SALEN((struct sockaddr *) addr), - (char*)pszHostFQDN, NI_MAXHOST, NULL, 0, NI_NAMEREQD); + fqdnBuf, NI_MAXHOST, NULL, 0, NI_NAMEREQD); -dbgprintf("dnscache: error %d after 2nd mygetnameinfo\n", error); if(error == 0) { memset (&hints, 0, sizeof (struct addrinfo)); hints.ai_flags = AI_NUMERICHOST; @@ -188,7 +291,7 @@ dbgprintf("dnscache: error %d after 2nd mygetnameinfo\n", error); * because we should not have obtained a non-numeric address. If * we got a numeric one, someone messed with DNS! */ - if(getaddrinfo ((char*)pszHostFQDN, NULL, &hints, &res) == 0) { + if(getaddrinfo (fqdnBuf, NULL, &hints, &res) == 0) { uchar szErrMsg[1024]; freeaddrinfo (res); /* OK, we know we have evil. The question now is what to do about @@ -204,7 +307,7 @@ dbgprintf("dnscache: error %d after 2nd mygetnameinfo\n", error); snprintf((char*)szErrMsg, sizeof(szErrMsg) / sizeof(uchar), "Malicious PTR record, message dropped " "IP = \"%s\" HOST = \"%s\"", - ip, pszHostFQDN); + szIP, fqdnBuf); errmsg.LogError(0, RS_RET_MALICIOUS_ENTITY, "%s", szErrMsg); pthread_sigmask(SIG_SETMASK, &omask, NULL); ABORT_FINALIZE(RS_RET_MALICIOUS_ENTITY); @@ -219,92 +322,76 @@ dbgprintf("dnscache: error %d after 2nd mygetnameinfo\n", error); snprintf((char*)szErrMsg, sizeof(szErrMsg) / sizeof(uchar), "Malicious PTR record (message accepted, but used IP " "instead of PTR name: IP = \"%s\" HOST = \"%s\"", - ip, pszHostFQDN); + szIP, fqdnBuf); errmsg.LogError(0, NO_ERRCODE, "%s", szErrMsg); error = 1; /* that will trigger using IP address below. */ + } else {/* we have a valid entry, so let's create the respective properties */ + fqdnLen = strlen(fqdnBuf); + prop.CreateStringProp(&etry->fqdn, (uchar*)fqdnBuf, fqdnLen); + for(i = 0 ; i < fqdnLen ; ++i) + fqdnBuf[i] = tolower(fqdnBuf[i]); + prop.CreateStringProp(&etry->fqdnLowerCase, (uchar*)fqdnBuf, fqdnLen); } } pthread_sigmask(SIG_SETMASK, &omask, NULL); } -dbgprintf("dnscache: error %d, DisableDNS %d\n", error, glbl.GetDisableDNS()); - if(error || glbl.GetDisableDNS()) { - dbgprintf("Host name for your address (%s) unknown\n", ip); - strcpy((char*) pszHostFQDN, (char*)ip); - } finalize_it: - RETiRet; -} + if(iRet != RS_RET_OK) { + strcpy(szIP, "?error.obtaining.ip?"); + error = 1; /* trigger hostname copies below! */ + } + /* we need to create the inputName property (only once during our lifetime) */ + prop.CreateStringProp(&etry->ip, (uchar*)szIP, strlen(szIP)); -/* evict an entry from the cache. We should try to evict one that does - * not decrease the hit rate that much, but we do not try to hard currently - * (as the base cache data structure may change). - * This MUST NOT be called when the cache is empty! - * rgerhards, 2011-06-06 - */ -static inline void -evictEntry(void) -{ - dnscache_entry_t *prev, *evict, *prevEvict, *etry; - unsigned lowest; - - prev = prevEvict = NULL; - evict = dnsCache.root; - lowest = evict->nUsed; - for(etry = dnsCache.root->next ; etry != NULL ; etry = etry->next) { - if(etry->nUsed < lowest) { - evict = etry; - lowest = etry->nUsed; - prevEvict = prev; - } - prev = etry; - } + if(error || glbl.GetDisableDNS()) { + dbgprintf("Host name for your address (%s) unknown\n", szIP); + prop.AddRef(etry->ip); + etry->fqdn = etry->ip; + prop.AddRef(etry->ip); + etry->fqdnLowerCase = etry->ip; + } - /* found lowest, unlink */ - if(prevEvict == NULL) { /* remove root? */ - dnsCache.root = evict->next; - } else { - prevEvict = evict->next; - } - entryDestruct(evict); + setLocalHostName(etry); + + RETiRet; } -/* add a new entry to the cache. This means the address is resolved and - * then added to the cache. - */ static inline rsRetVal addEntry(struct sockaddr_storage *addr, dnscache_entry_t **pEtry) { - uchar pszHostFQDN[NI_MAXHOST]; - uchar ip[80]; /* 80 is safe for larges IPv6 addr */ - dnscache_entry_t *etry; + int r; + struct sockaddr_storage *keybuf; + dnscache_entry_t *etry = NULL; DEFiRet; - CHKiRet(resolveAddr(addr, pszHostFQDN, ip)); + CHKmalloc(etry = MALLOC(sizeof(dnscache_entry_t))); - CHKmalloc(etry->pszHostFQDN = ustrdup(pszHostFQDN)); - CHKmalloc(etry->ip = ustrdup(ip)); + CHKiRet(resolveAddr(addr, etry)); memcpy(&etry->addr, addr, SALEN((struct sockaddr*) addr)); etry->nUsed = 0; *pEtry = etry; - /* add to list. Currently, we place the new element always at - * the root node. This needs to be optimized later. 2011-06-06 - */ + CHKmalloc(keybuf = malloc(sizeof(struct sockaddr_storage))); + memcpy(keybuf, addr, sizeof(struct sockaddr_storage)); + pthread_rwlock_unlock(&dnsCache.rwlock); /* release read lock */ pthread_rwlock_wrlock(&dnsCache.rwlock); /* and re-aquire for writing */ - if(dnsCache.nEntries >= MAX_CACHE_ENTRIES) { - evictEntry(); + r = hashtable_insert(dnsCache.ht, keybuf, *pEtry); + if(r == 0) { + DBGPRINTF("dnscache: inserting element failed\n"); } - etry->next = dnsCache.root; - dnsCache.root = etry; pthread_rwlock_unlock(&dnsCache.rwlock); - pthread_rwlock_rdlock(&dnsCache.rwlock); /* TODO: optimize this! */ + pthread_rwlock_rdlock(&dnsCache.rwlock); /* we need this again */ finalize_it: + if(iRet != RS_RET_OK && etry != NULL) { + /* Note: sub-fields cannot be populated in this case */ + free(etry); + } RETiRet; } @@ -314,7 +401,7 @@ finalize_it: * TODO: implement! */ static inline rsRetVal -validateEntry(dnscache_entry_t *etry, struct sockaddr_storage *addr) +validateEntry(dnscache_entry_t __attribute__((unused)) *etry, struct sockaddr_storage __attribute__((unused)) *addr) { return RS_RET_OK; } @@ -322,10 +409,12 @@ validateEntry(dnscache_entry_t *etry, struct sockaddr_storage *addr) /* This is the main function: it looks up an entry and returns it's name * and IP address. If the entry is not yet inside the cache, it is added. - * If the entry can not be resolved, an error is reported back. + * If the entry can not be resolved, an error is reported back. If fqdn + * or fqdnLowerCase are NULL, they are not set. */ rsRetVal -dnscacheLookup(struct sockaddr_storage *addr, uchar *pszHostFQDN, uchar *ip) +dnscacheLookup(struct sockaddr_storage *addr, prop_t **fqdn, prop_t **fqdnLowerCase, + prop_t **localName, prop_t **ip) { dnscache_entry_t *etry; DEFiRet; @@ -338,18 +427,39 @@ dnscacheLookup(struct sockaddr_storage *addr, uchar *pszHostFQDN, uchar *ip) } else { CHKiRet(validateEntry(etry, addr)); } - // TODO/QUESTION: can we get rid of the strcpy? -dbgprintf("XXXX: hostn '%s', ip '%s'\n", etry->pszHostFQDN, etry->ip); - strcpy((char*)pszHostFQDN, (char*)etry->pszHostFQDN); - strcpy((char*)ip, (char*)etry->ip); + prop.AddRef(etry->ip); + *ip = etry->ip; + if(fqdn != NULL) { + prop.AddRef(etry->fqdn); + *fqdn = etry->fqdn; + } + if(fqdnLowerCase != NULL) { + prop.AddRef(etry->fqdnLowerCase); + *fqdnLowerCase = etry->fqdnLowerCase; + } + if(localName != NULL) { + prop.AddRef(etry->localName); + *localName = etry->localName; + } finalize_it: pthread_rwlock_unlock(&dnsCache.rwlock); -dbgprintf("XXXX: dnscacheLookup finished, iRet=%d\n", iRet); if(iRet != RS_RET_OK && iRet != RS_RET_ADDRESS_UNKNOWN) { DBGPRINTF("dnscacheLookup failed with iRet %d\n", iRet); - strcpy((char*) pszHostFQDN, "???"); - strcpy((char*) ip, "???"); + prop.AddRef(staticErrValue); + *ip = staticErrValue; + if(fqdn != NULL) { + prop.AddRef(staticErrValue); + *fqdn = staticErrValue; + } + if(fqdnLowerCase != NULL) { + prop.AddRef(staticErrValue); + *fqdnLowerCase = staticErrValue; + } + if(localName != NULL) { + prop.AddRef(staticErrValue); + *localName = staticErrValue; + } } RETiRet; } diff --git a/runtime/dnscache.h b/runtime/dnscache.h index 69f038ee..9c21a645 100644 --- a/runtime/dnscache.h +++ b/runtime/dnscache.h @@ -1,6 +1,6 @@ /* Definitions for dnscache module. * - * Copyright 2011-2012 Adiscon GmbH. + * Copyright 2011-2013 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -24,6 +24,6 @@ rsRetVal dnscacheInit(void); rsRetVal dnscacheDeinit(void); -rsRetVal dnscacheLookup(struct sockaddr_storage *addr, uchar *pszHostFQDN, uchar *ip); +rsRetVal dnscacheLookup(struct sockaddr_storage *addr, prop_t **fqdn, prop_t **fqdnLowerCase, prop_t **localName, prop_t **ip); #endif /* #ifndef INCLUDED_DNSCACHE_H */ diff --git a/runtime/glbl.c b/runtime/glbl.c index a0997829..b3fe3a1d 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -7,7 +7,7 @@ * * Module begun 2008-04-16 by Rainer Gerhards * - * Copyright 2008-2011 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -82,6 +82,7 @@ static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm d static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */ static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */ static int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ +pid_t glbl_ourpid; #ifndef HAVE_ATOMIC_BUILTINS static DEF_ATOMIC_HELPER_MUT(mutTerminateInputs); #endif @@ -210,7 +211,7 @@ setLocalHostIPIF(void __attribute__((unused)) *pVal, uchar *pNewVal) if(propLocalIPIF != NULL) { errmsg.LogError(0, RS_RET_ERR, "$LocalHostIPIF is already set " "and cannot be reset; place it at TOP OF rsyslog.conf!"); - ABORT_FINALIZE(RS_RET_ERR_WRKDIR); + ABORT_FINALIZE(RS_RET_ERR); } localRet = net.GetIFIPAddr(pNewVal, AF_UNSPEC, myIP, (int) sizeof(myIP)); @@ -278,6 +279,28 @@ finalize_it: RETiRet; } + +static rsRetVal +setDebugFile(void __attribute__((unused)) *pVal, uchar *pNewVal) +{ + DEFiRet; + dbgSetDebugFile(pNewVal); + free(pNewVal); + RETiRet; +} + + +static rsRetVal +setDebugLevel(void __attribute__((unused)) *pVal, int level) +{ + DEFiRet; + dbgSetDebugLevel(level); + dbgprintf("debug level %d set via config file\n", level); + dbgprintf("This is rsyslog version " VERSION "\n"); + RETiRet; +} + + /* return our local IP. * If no local IP is set, "127.0.0.1" is selected *and* set. This * is an intensional side effect that we do in order to keep things @@ -610,6 +633,8 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* config handlers are never unregistered and need not be - we are always loaded ;) */ + CHKiRet(regCfSysLineHdlr((uchar *)"debugfile", 0, eCmdHdlrGetWord, setDebugFile, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"debuglevel", 0, eCmdHdlrInt, setDebugLevel, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, setWorkDir, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"dropmsgswithmaliciousdnsptrrecords", 0, eCmdHdlrBinary, NULL, &bDropMalPTRMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvr, NULL)); diff --git a/runtime/glbl.h b/runtime/glbl.h index d2d1e66a..e95e48f7 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -30,11 +30,14 @@ #ifndef GLBL_H_INCLUDED #define GLBL_H_INCLUDED +#include <sys/types.h> #include "rainerscript.h" #include "prop.h" #define glblGetIOBufSize() 4096 /* size of the IO buffer, e.g. for strm class */ +extern pid_t glbl_ourpid; + /* interfaces */ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ uchar* (*GetWorkDir)(void); @@ -86,6 +89,9 @@ ENDinterface(glbl) /* the remaining prototypes */ PROTOTYPEObj(glbl); +static inline pid_t glblGetOurPid(void) { return glbl_ourpid; } +static inline void glblSetOurPid(pid_t pid) { glbl_ourpid = pid; } + void glblPrepCnf(void); void glblProcessCnf(struct cnfobj *o); void glblDoneLoadCnf(void); diff --git a/runtime/hashtable.c b/runtime/hashtable.c index a01fa7d9..f718bd43 100644 --- a/runtime/hashtable.c +++ b/runtime/hashtable.c @@ -263,7 +263,7 @@ hashtable_destroy(struct hashtable *h, int free_values) /* some generic hash functions */ -/* one provided by Aaaron Wiebe based on perl's hashng algorithm +/* one provided by Aaaron Wiebe based on perl's hashing algorithm * (so probably pretty generic). Not for excessively large strings! */ unsigned int diff --git a/runtime/libgcry.c b/runtime/libgcry.c new file mode 100644 index 00000000..51c10af4 --- /dev/null +++ b/runtime/libgcry.c @@ -0,0 +1,426 @@ +/* gcry.c - rsyslog's libgcrypt based crypto provider + * + * Copyright 2013 Adiscon GmbH. + * + * We need to store some additional information in support of encryption. + * For this, we create a side-file, which is named like the actual log + * file, but with the suffix ".encinfo" appended. It contains the following + * records: + * IV:<hex> The initial vector used at block start. Also indicates start + * start of block. + * END:<int> The end offset of the block, as uint64_t in decimal notation. + * This is used during encryption to know when the current + * encryption block ends. + * For the current implementation, there must always be an IV record + * followed by an END record. Each records is LF-terminated. Record + * types can simply be extended in the future by specifying new + * types (like "IV") before the colon. + * To identify a file as rsyslog encryption info file, it must start with + * the line "FILETYPE:rsyslog-enrcyption-info" + * There are some size constraints: the recordtype must be 31 bytes at + * most and the actual value (between : and LF) must be 1023 bytes at most. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include <stdio.h> +#include <gcrypt.h> +#include <sys/stat.h> +#include <sys/uio.h> +#include <sys/types.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> + +#include "rsyslog.h" +#include "libgcry.h" + + +static rsRetVal +eiWriteRec(gcryfile gf, char *recHdr, size_t lenRecHdr, char *buf, size_t lenBuf) +{ + struct iovec iov[3]; + ssize_t nwritten, towrite; + DEFiRet; + + iov[0].iov_base = recHdr; + iov[0].iov_len = lenRecHdr; + iov[1].iov_base = buf; + iov[1].iov_len = lenBuf; + iov[2].iov_base = "\n"; + iov[2].iov_len = 1; + towrite = iov[0].iov_len + iov[1].iov_len + iov[2].iov_len; + nwritten = writev(gf->fd, iov, sizeof(iov)/sizeof(struct iovec)); + if(nwritten != towrite) { + DBGPRINTF("eiWrite%s: error writing file, towrite %d, " + "nwritten %d\n", recHdr, (int) towrite, (int) nwritten); + ABORT_FINALIZE(RS_RET_EI_WR_ERR); + } + DBGPRINTF("encryption info file %s: written %s, len %d\n", + recHdr, gf->eiName, (int) nwritten); +finalize_it: + RETiRet; +} + +static rsRetVal +eiOpenRead(gcryfile gf) +{ + DEFiRet; + gf->fd = open((char*)gf->eiName, O_RDONLY|O_NOCTTY|O_CLOEXEC); + if(gf->fd == -1) { + ABORT_FINALIZE(errno == ENOENT ? RS_RET_EI_NO_EXISTS : RS_RET_EI_OPN_ERR); + } +finalize_it: + RETiRet; +} + + +static rsRetVal +eiCheckFiletype(gcryfile gf) +{ + char hdrBuf[128]; + size_t toRead, didRead; + DEFiRet; + + CHKiRet(eiOpenRead(gf)); + if(Debug) memset(hdrBuf, 0, sizeof(hdrBuf)); /* for dbgprintf below! */ + toRead = sizeof("FILETYPE:")-1 + sizeof(RSGCRY_FILETYPE_NAME)-1 + 1; + didRead = read(gf->fd, hdrBuf, toRead); + close(gf->fd); + DBGPRINTF("eiCheckFiletype read %d bytes: '%s'\n", didRead, hdrBuf); + if( didRead != toRead + || strncmp(hdrBuf, "FILETYPE:" RSGCRY_FILETYPE_NAME "\n", toRead)) + iRet = RS_RET_EI_INVLD_FILE; +finalize_it: + RETiRet; +} + +static rsRetVal +eiOpenAppend(gcryfile gf) +{ + rsRetVal localRet; + DEFiRet; + localRet = eiCheckFiletype(gf); + if(localRet == RS_RET_OK) { + gf->fd = open((char*)gf->eiName, + O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC, 0600); + if(gf->fd == -1) { + ABORT_FINALIZE(RS_RET_EI_OPN_ERR); + } + } else if(localRet == RS_RET_EI_NO_EXISTS) { + /* looks like we need to create a new file */ + gf->fd = open((char*)gf->eiName, + O_WRONLY|O_CREAT|O_NOCTTY|O_CLOEXEC, 0600); + if(gf->fd == -1) { + ABORT_FINALIZE(RS_RET_EI_OPN_ERR); + } + CHKiRet(eiWriteRec(gf, "FILETYPE:", 9, RSGCRY_FILETYPE_NAME, + sizeof(RSGCRY_FILETYPE_NAME)-1)); + } else { + gf->fd = -1; + ABORT_FINALIZE(localRet); + } + DBGPRINTF("encryption info file %s: opened as #%d\n", + gf->eiName, gf->fd); +finalize_it: + RETiRet; +} + +static rsRetVal +eiWriteIV(gcryfile gf, uchar *iv) +{ + static const char hexchars[16] = + {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; + unsigned iSrc, iDst; + char hex[4096]; + DEFiRet; + + if(gf->blkLength > sizeof(hex)/2) { + DBGPRINTF("eiWriteIV: crypto block len way too large, aborting " + "write"); + ABORT_FINALIZE(RS_RET_ERR); + } + + for(iSrc = iDst = 0 ; iSrc < gf->blkLength ; ++iSrc) { + hex[iDst++] = hexchars[iv[iSrc]>>4]; + hex[iDst++] = hexchars[iv[iSrc]&0x0f]; + } + + iRet = eiWriteRec(gf, "IV:", 3, hex, gf->blkLength*2); +finalize_it: + RETiRet; +} + +/* we do not return an error state, as we MUST close the file, + * no matter what happens. + */ +static void +eiClose(gcryfile gf, off64_t offsLogfile) +{ + char offs[21]; + size_t len; + if(gf->fd == -1) + return; + /* 2^64 is 20 digits, so the snprintf buffer is large enough */ + len = snprintf(offs, sizeof(offs), "%lld", offsLogfile); + eiWriteRec(gf, "END:", 4, offs, len); + close(gf->fd); + DBGPRINTF("encryption info file %s: closed\n", gf->eiName); +} + +static rsRetVal +gcryfileConstruct(gcryctx ctx, gcryfile *pgf, uchar *logfn) +{ + char fn[MAXFNAME+1]; + gcryfile gf; + DEFiRet; + + CHKmalloc(gf = calloc(1, sizeof(struct gcryfile_s))); + gf->ctx = ctx; + snprintf(fn, sizeof(fn), "%s%s", logfn, ENCINFO_SUFFIX); + fn[MAXFNAME] = '\0'; /* be on save side */ + gf->eiName = (uchar*) strdup(fn); + *pgf = gf; +finalize_it: + RETiRet; +} + + +gcryctx +gcryCtxNew(void) +{ + gcryctx ctx; + ctx = calloc(1, sizeof(struct gcryctx_s)); + ctx->algo = GCRY_CIPHER_AES128; + ctx->mode = GCRY_CIPHER_MODE_CBC; + return ctx; +} + +int +gcryfileDestruct(gcryfile gf, off64_t offsLogfile) +{ + int r = 0; + if(gf == NULL) + goto done; + + eiClose(gf, offsLogfile); + free(gf->eiName); + free(gf); +done: return r; +} +void +rsgcryCtxDel(gcryctx ctx) +{ + if(ctx != NULL) { + free(ctx); + } +} + +static inline void +addPadding(gcryfile pF, uchar *buf, size_t *plen) +{ + unsigned i; + size_t nPad; + nPad = (pF->blkLength - *plen % pF->blkLength) % pF->blkLength; + DBGPRINTF("libgcry: addPadding %d chars, blkLength %d, mod %d, pad %d\n", + *plen, pF->blkLength, *plen % pF->blkLength, nPad); + for(i = 0 ; i < nPad ; ++i) + buf[(*plen)+i] = 0x00; + (*plen)+= nPad; +} + +static inline void +removePadding(char *buf, size_t *plen) +{ + unsigned len = (unsigned) *plen; + unsigned iSrc, iDst; + char *frstNUL; + + frstNUL = strchr(buf, 0x00); + if(frstNUL == NULL) + goto done; + iDst = iSrc = frstNUL - buf; + + while(iSrc < len) { + if(buf[iSrc] != 0x00) + buf[iDst++] = buf[iSrc]; + ++iSrc; + } + + *plen = iDst; +done: return; +} + +/* returns 0 on succes, positive if key length does not match and key + * of return value size is required. + */ +int +rsgcrySetKey(gcryctx ctx, unsigned char *key, uint16_t keyLen) +{ + uint16_t reqKeyLen; + int r; + + reqKeyLen = gcry_cipher_get_algo_keylen(ctx->algo); + if(keyLen != reqKeyLen) { + r = reqKeyLen; + goto done; + } + ctx->keyLen = keyLen; + ctx->key = malloc(keyLen); + memcpy(ctx->key, key, keyLen); + r = 0; +done: return r; +} + +rsRetVal +rsgcrySetMode(gcryctx ctx, uchar *modename) +{ + int mode; + DEFiRet; + + mode = rsgcryModename2Mode((char *)modename); + if(mode == GCRY_CIPHER_MODE_NONE) { + ABORT_FINALIZE(RS_RET_CRY_INVLD_MODE); + } + ctx->mode = mode; +finalize_it: + RETiRet; +} + +rsRetVal +rsgcrySetAlgo(gcryctx ctx, uchar *algoname) +{ + int algo; + DEFiRet; + + algo = rsgcryAlgoname2Algo((char *)algoname); + if(algo == GCRY_CIPHER_NONE) { + ABORT_FINALIZE(RS_RET_CRY_INVLD_ALGO); + } + ctx->algo = algo; +finalize_it: + RETiRet; +} + +/* As of some Linux and security expert I spoke to, /dev/urandom + * provides very strong random numbers, even if it runs out of + * entropy. As far as he knew, this is save for all applications + * (and he had good proof that I currently am not permitted to + * reproduce). -- rgerhards, 2013-03-04 + */ +void +seedIV(gcryfile gf, uchar **iv) +{ + int fd; + + *iv = malloc(gf->blkLength); /* do NOT zero-out! */ + /* if we cannot obtain data from /dev/urandom, we use whatever + * is present at the current memory location as random data. Of + * course, this is very weak and we should consider a different + * option, especially when not running under Linux (for Linux, + * unavailability of /dev/urandom is just a theoretic thing, it + * will always work...). -- TODO -- rgerhards, 2013-03-06 + */ + if((fd = open("/dev/urandom", O_RDONLY)) > 0) { + if(read(fd, *iv, gf->blkLength)) {}; /* keep compiler happy */ + close(fd); + } +} + +rsRetVal +rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, uchar *fname) +{ + gcry_error_t gcryError; + gcryfile gf = NULL; + uchar *iv = NULL; + DEFiRet; + + CHKiRet(gcryfileConstruct(ctx, &gf, fname)); + + gf->blkLength = gcry_cipher_get_algo_blklen(ctx->algo); + + gcryError = gcry_cipher_open(&gf->chd, ctx->algo, ctx->mode, 0); + if (gcryError) { + dbgprintf("gcry_cipher_open failed: %s/%s\n", + gcry_strsource(gcryError), + gcry_strerror(gcryError)); + ABORT_FINALIZE(RS_RET_ERR); + } + + gcryError = gcry_cipher_setkey(gf->chd, gf->ctx->key, gf->ctx->keyLen); + if (gcryError) { + dbgprintf("gcry_cipher_setkey failed: %s/%s\n", + gcry_strsource(gcryError), + gcry_strerror(gcryError)); + ABORT_FINALIZE(RS_RET_ERR); + } + + seedIV(gf, &iv); + gcryError = gcry_cipher_setiv(gf->chd, iv, gf->blkLength); + if (gcryError) { + dbgprintf("gcry_cipher_setiv failed: %s/%s\n", + gcry_strsource(gcryError), + gcry_strerror(gcryError)); + ABORT_FINALIZE(RS_RET_ERR); + } + CHKiRet(eiOpenAppend(gf)); + CHKiRet(eiWriteIV(gf, iv)); + *pgf = gf; +finalize_it: + free(iv); + if(iRet != RS_RET_OK && gf != NULL) + gcryfileDestruct(gf, -1); + RETiRet; +} + +int +rsgcryEncrypt(gcryfile pF, uchar *buf, size_t *len) +{ + int gcryError; + DEFiRet; + + if(*len == 0) + FINALIZE; + + addPadding(pF, buf, len); + gcryError = gcry_cipher_encrypt(pF->chd, buf, *len, NULL, 0); + if(gcryError) { + dbgprintf("gcry_cipher_encrypt failed: %s/%s\n", + gcry_strsource(gcryError), + gcry_strerror(gcryError)); + ABORT_FINALIZE(RS_RET_ERR); + } +finalize_it: + RETiRet; +} + + +/* module-init dummy for potential later use */ +int +rsgcryInit(void) +{ + return 0; +} + +/* module-deinit dummy for potential later use */ +void +rsgcryExit(void) +{ + return; +} diff --git a/runtime/libgcry.h b/runtime/libgcry.h new file mode 100644 index 00000000..b77b0f9e --- /dev/null +++ b/runtime/libgcry.h @@ -0,0 +1,101 @@ +/* libgcry.h - rsyslog's guardtime support library + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_LIBGCRY_H +#define INCLUDED_LIBGCRY_H +#include <stdint.h> + + +struct gcryctx_s { + uchar *key; + size_t keyLen; + int algo; + int mode; +}; +typedef struct gcryctx_s *gcryctx; +typedef struct gcryfile_s *gcryfile; + +/* this describes a file, as far as libgcry is concerned */ +struct gcryfile_s { + gcry_cipher_hd_t chd; /* cypher handle */ + size_t blkLength; /* size of low-level crypto block */ + uchar *eiName; /* name of .encinfo file */ + int fd; /* descriptor of .encinfo file (-1 if not open) */ + gcryctx ctx; +}; + +int gcryGetKeyFromFile(char *fn, char **key, unsigned *keylen); +int rsgcryInit(void); +void rsgcryExit(void); +int rsgcrySetKey(gcryctx ctx, unsigned char *key, uint16_t keyLen); +rsRetVal rsgcrySetMode(gcryctx ctx, uchar *algoname); +rsRetVal rsgcrySetAlgo(gcryctx ctx, uchar *modename); +gcryctx gcryCtxNew(void); +void rsgcryCtxDel(gcryctx ctx); +int gcryfileDestruct(gcryfile gf, off64_t offsLogfile); +rsRetVal rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, uchar *fname); +int rsgcryEncrypt(gcryfile pF, uchar *buf, size_t *len); + +/* error states */ +#define RSGCRYE_EI_OPEN 1 /* error opening .encinfo file */ +#define RSGCRYE_OOM 4 /* ran out of memory */ + +#define EIF_MAX_RECTYPE_LEN 31 /* max length of record types */ +#define EIF_MAX_VALUE_LEN 1023 /* max length of value types */ +#define RSGCRY_FILETYPE_NAME "rsyslog-enrcyption-info" +#define ENCINFO_SUFFIX ".encinfo" + +static inline int +rsgcryAlgoname2Algo(char *algoname) { + if(!strcmp((char*)algoname, "3DES")) return GCRY_CIPHER_3DES; + if(!strcmp((char*)algoname, "CAST5")) return GCRY_CIPHER_CAST5; + if(!strcmp((char*)algoname, "BLOWFISH")) return GCRY_CIPHER_BLOWFISH; + if(!strcmp((char*)algoname, "AES128")) return GCRY_CIPHER_AES128; + if(!strcmp((char*)algoname, "AES192")) return GCRY_CIPHER_AES192; + if(!strcmp((char*)algoname, "AES256")) return GCRY_CIPHER_AES256; + if(!strcmp((char*)algoname, "TWOFISH")) return GCRY_CIPHER_TWOFISH; + if(!strcmp((char*)algoname, "TWOFISH128")) return GCRY_CIPHER_TWOFISH128; + if(!strcmp((char*)algoname, "ARCFOUR")) return GCRY_CIPHER_ARCFOUR; + if(!strcmp((char*)algoname, "DES")) return GCRY_CIPHER_DES; + if(!strcmp((char*)algoname, "SERPENT128")) return GCRY_CIPHER_SERPENT128; + if(!strcmp((char*)algoname, "SERPENT192")) return GCRY_CIPHER_SERPENT192; + if(!strcmp((char*)algoname, "SERPENT256")) return GCRY_CIPHER_SERPENT256; + if(!strcmp((char*)algoname, "RFC2268_40")) return GCRY_CIPHER_RFC2268_40; + if(!strcmp((char*)algoname, "SEED")) return GCRY_CIPHER_SEED; + if(!strcmp((char*)algoname, "CAMELLIA128")) return GCRY_CIPHER_CAMELLIA128; + if(!strcmp((char*)algoname, "CAMELLIA192")) return GCRY_CIPHER_CAMELLIA192; + if(!strcmp((char*)algoname, "CAMELLIA256")) return GCRY_CIPHER_CAMELLIA256; + return GCRY_CIPHER_NONE; +} + +static inline int +rsgcryModename2Mode(char *modename) { + if(!strcmp((char*)modename, "ECB")) return GCRY_CIPHER_MODE_ECB; + if(!strcmp((char*)modename, "CFB")) return GCRY_CIPHER_MODE_CFB; + if(!strcmp((char*)modename, "CBC")) return GCRY_CIPHER_MODE_CBC; + if(!strcmp((char*)modename, "STREAM")) return GCRY_CIPHER_MODE_STREAM; + if(!strcmp((char*)modename, "OFB")) return GCRY_CIPHER_MODE_OFB; + if(!strcmp((char*)modename, "CTR")) return GCRY_CIPHER_MODE_CTR; +# ifdef GCRY_CIPHER_MODE_AESWRAP + if(!strcmp((char*)modename, "AESWRAP")) return GCRY_CIPHER_MODE_AESWRAP; +# endif + return GCRY_CIPHER_MODE_NONE; +} +#endif /* #ifndef INCLUDED_LIBGCRY_H */ diff --git a/runtime/libgcry_common.c b/runtime/libgcry_common.c new file mode 100644 index 00000000..07a524dc --- /dev/null +++ b/runtime/libgcry_common.c @@ -0,0 +1,206 @@ +/* libgcry_common.c + * This file hosts functions both being used by the rsyslog runtime as + * well as tools who do not use the runtime (so we can maintain the + * code at a single place). + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include <stdio.h> +#include <gcrypt.h> +#include <sys/stat.h> +#include <sys/uio.h> +#include <sys/types.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> + +#include "rsyslog.h" /* we need data typedefs */ +#include "libgcry.h" + + +/* read a key from a key file + * @param[out] key - key buffer, must be freed by caller + * @param[out] keylen - length of buffer + * @returns 0 if OK, something else otherwise (we do not use + * iRet as this is also called from non-rsyslog w/o runtime) + * The key length is limited to 64KiB to prevent DoS. + * Note well: key is a blob, not a C string (NUL may be present!) + */ +int +gcryGetKeyFromFile(char *fn, char **key, unsigned *keylen) +{ + struct stat sb; + int fd; + int r; + + if(stat(fn, &sb) == -1) { + r = 1; goto done; + } + if((sb.st_mode & S_IFMT) != S_IFREG) { + r = 2; goto done; + } + if(sb.st_size > 64*1024) { + r = 3; goto done; + } + if((*key = malloc(sb.st_size)) == NULL) { + r = -1; goto done; + } + if((fd = open(fn, O_RDONLY)) < 0) { + r = 4; goto done; + } + if(read(fd, *key, sb.st_size) != sb.st_size) { + r = 5; goto done; + } + *keylen = sb.st_size; + close(fd); + r = 0; +done: return r; +} + + +/* execute the child process (must be called in child context + * after fork). + */ + +static void +execKeyScript(char *cmd, int pipefd[]) +{ + char *newargv[] = { NULL }; + char *newenviron[] = { NULL }; + + dup2(pipefd[0], STDIN_FILENO); + dup2(pipefd[1], STDOUT_FILENO); + + /* finally exec child */ +fprintf(stderr, "pre execve: %s\n", cmd); + execve(cmd, newargv, newenviron); + /* switch to? + execlp((char*)program, (char*) program, (char*)arg, NULL); + */ + + /* we should never reach this point, but if we do, we terminate */ + return; +} + + +static int +openPipe(char *cmd, int *fd) +{ + int pipefd[2]; + pid_t cpid; + int r; + + if(pipe(pipefd) == -1) { + r = 1; goto done; + } + + cpid = fork(); + if(cpid == -1) { + r = 1; goto done; + } + + if(cpid == 0) { + /* we are the child */ + execKeyScript(cmd, pipefd); + exit(1); + } + + close(pipefd[1]); + *fd = pipefd[0]; + r = 0; +done: return r; +} + + +/* Read a character from the program's output. */ +// TODO: highly unoptimized version, should be used in buffered +// mode +static int +readProgChar(int fd, char *c) +{ + int r; + if(read(fd, c, 1) != 1) { + r = 1; goto done; + } + r = 0; +done: return r; +} + +/* Read a line from the script. Line is terminated by LF, which + * is NOT put into the buffer. + * buf must be 64KiB + */ +static int +readProgLine(int fd, char *buf) +{ + char c; + int r; + unsigned i; + + for(i = 0 ; i < 64*1024 ; ++i) { + if((r = readProgChar(fd, &c)) != 0) goto done; + if(c == '\n') + break; + buf[i] = c; + }; + if(i >= 64*1024) { + r = 1; goto done; + } + buf[i] = '\0'; + r = 0; +done: return r; +} +static int +readProgKey(int fd, char *buf, unsigned keylen) +{ + char c; + int r; + unsigned i; + + for(i = 0 ; i < keylen ; ++i) { + if((r = readProgChar(fd, &c)) != 0) goto done; + buf[i] = c; + }; + r = 0; +done: return r; +} + +int +gcryGetKeyFromProg(char *cmd, char **key, unsigned *keylen) +{ + int r; + int fd; + char rcvBuf[64*1024]; + + if((r = openPipe(cmd, &fd)) != 0) goto done; + if((r = readProgLine(fd, rcvBuf)) != 0) goto done; + if(strcmp(rcvBuf, "RSYSLOG-KEY-PROVIDER:0")) { + r = 2; goto done; + } + if((r = readProgLine(fd, rcvBuf)) != 0) goto done; + *keylen = atoi(rcvBuf); + if((*key = malloc(*keylen)) == NULL) { + r = -1; goto done; + } + if((r = readProgKey(fd, *key, *keylen)) != 0) goto done; +done: return r; +} diff --git a/runtime/librsgt.c b/runtime/librsgt.c new file mode 100644 index 00000000..85fc7742 --- /dev/null +++ b/runtime/librsgt.c @@ -0,0 +1,845 @@ +/* librsgt.c - rsyslog's guardtime support library + * + * Regarding the online algorithm for Merkle tree signing. Expected + * calling sequence is: + * + * sigblkConstruct + * for each signature block: + * sigblkInit + * for each record: + * sigblkAddRecord + * sigblkFinish + * sigblkDestruct + * + * Obviously, the next call after sigblkFinsh must either be to + * sigblkInit or sigblkDestruct (if no more signature blocks are + * to be emitted, e.g. on file close). sigblkDestruct saves state + * information (most importantly last block hash) and sigblkConstruct + * reads (or initilizes if not present) it. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <stdint.h> +#include <assert.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#define MAXFNAME 1024 + +#include <gt_http.h> + +#include "librsgt.h" + +typedef unsigned char uchar; +#ifndef VERSION +#define VERSION "no-version" +#endif + + +static void +reportErr(gtctx ctx, char *errmsg) +{ + if(ctx->errFunc == NULL) + goto done; + ctx->errFunc(ctx->usrptr, (uchar*)errmsg); +done: return; +} + +static void +reportGTAPIErr(gtctx ctx, gtfile gf, char *apiname, int ecode) +{ + char errbuf[4096]; + snprintf(errbuf, sizeof(errbuf), "%s[%s:%d]: %s", + (gf == NULL) ? (uchar*)"" : gf->sigfilename, + apiname, ecode, GT_getErrorString(ecode)); + errbuf[sizeof(errbuf)-1] = '\0'; + reportErr(ctx, errbuf); +} + +void +rsgtsetErrFunc(gtctx ctx, void (*func)(void*, uchar *), void *usrptr) +{ + ctx->usrptr = usrptr; + ctx->errFunc = func; +} + +imprint_t * +rsgtImprintFromGTDataHash(GTDataHash *hash) +{ + imprint_t *imp; + + if((imp = calloc(1, sizeof(imprint_t))) == NULL) { + goto done; + } + imp->hashID = hashIdentifier(hash->algorithm), + imp->len = hash->digest_length; + if((imp->data = (uint8_t*)malloc(imp->len)) == NULL) { + free(imp); imp = NULL; goto done; + } + memcpy(imp->data, hash->digest, imp->len); +done: return imp; +} + +void +rsgtimprintDel(imprint_t *imp) +{ + if(imp != NULL) { + free(imp->data), + free(imp); + } +} + +int +rsgtInit(char *usragent) +{ + int r = 0; + int ret = GT_OK; + + ret = GT_init(); + if(ret != GT_OK) { + r = 1; + goto done; + } + ret = GTHTTP_init(usragent, 1); + if(ret != GT_OK) { + r = 1; + goto done; + } +done: return r; +} + +void +rsgtExit(void) +{ + GTHTTP_finalize(); + GT_finalize(); +} + + +static inline gtfile +rsgtfileConstruct(gtctx ctx) +{ + gtfile gf; + if((gf = calloc(1, sizeof(struct gtfile_s))) == NULL) + goto done; + gf->ctx = ctx; + gf->hashAlg = ctx->hashAlg; + gf->blockSizeLimit = ctx->blockSizeLimit; + gf->bKeepRecordHashes = ctx->bKeepRecordHashes; + gf->bKeepTreeHashes = ctx->bKeepTreeHashes; + gf->x_prev = NULL; + +done: return gf; +} + +static inline int +tlvbufPhysWrite(gtfile gf) +{ + ssize_t lenBuf; + ssize_t iTotalWritten; + ssize_t iWritten; + char *pWriteBuf; + int r = 0; + + lenBuf = gf->tlvIdx; + pWriteBuf = gf->tlvBuf; + iTotalWritten = 0; + do { + iWritten = write(gf->fd, pWriteBuf, lenBuf); + if(iWritten < 0) { + iWritten = 0; /* we have written NO bytes! */ + if(errno == EINTR) { + /*NO ERROR, just continue */; + } else { + reportErr(gf->ctx, "signature file write error"); + r = RSGTE_IO; + goto finalize_it; + } + } + /* advance buffer to next write position */ + iTotalWritten += iWritten; + lenBuf -= iWritten; + pWriteBuf += iWritten; + } while(lenBuf > 0); /* Warning: do..while()! */ + +finalize_it: + gf->tlvIdx = 0; + return r; +} + +static inline int +tlvbufChkWrite(gtfile gf) +{ + if(gf->tlvIdx == sizeof(gf->tlvBuf)) { + return tlvbufPhysWrite(gf); + } + return 0; +} + + +/* write to TLV file buffer. If buffer is full, an actual call occurs. Else + * output is written only on flush or close. + */ +static inline int +tlvbufAddOctet(gtfile gf, int8_t octet) +{ + int r; + r = tlvbufChkWrite(gf); + if(r != 0) goto done; + gf->tlvBuf[gf->tlvIdx++] = octet; +done: return r; +} +static inline int +tlvbufAddOctetString(gtfile gf, uint8_t *octet, int size) +{ + int i, r = 0; + for(i = 0 ; i < size ; ++i) { + r = tlvbufAddOctet(gf, octet[i]); + if(r != 0) goto done; + } +done: return r; +} +/* return the actual length in to-be-written octets of an integer */ +static inline uint8_t +tlvbufGetInt64OctetSize(uint64_t val) +{ + if(val >> 56) + return 8; + if((val >> 48) & 0xff) + return 7; + if((val >> 40) & 0xff) + return 6; + if((val >> 32) & 0xff) + return 5; + if((val >> 24) & 0xff) + return 4; + if((val >> 16) & 0xff) + return 3; + if((val >> 8) & 0xff) + return 2; + return 1; +} +static inline int +tlvbufAddInt64(gtfile gf, uint64_t val) +{ + uint8_t doWrite = 0; + int r; + if(val >> 56) { + r = tlvbufAddOctet(gf, (val >> 56) & 0xff), doWrite = 1; + if(r != 0) goto done; + } + if(doWrite || ((val >> 48) & 0xff)) { + r = tlvbufAddOctet(gf, (val >> 48) & 0xff), doWrite = 1; + if(r != 0) goto done; + } + if(doWrite || ((val >> 40) & 0xff)) { + r = tlvbufAddOctet(gf, (val >> 40) & 0xff), doWrite = 1; + if(r != 0) goto done; + } + if(doWrite || ((val >> 32) & 0xff)) { + r = tlvbufAddOctet(gf, (val >> 32) & 0xff), doWrite = 1; + if(r != 0) goto done; + } + if(doWrite || ((val >> 24) & 0xff)) { + r = tlvbufAddOctet(gf, (val >> 24) & 0xff), doWrite = 1; + if(r != 0) goto done; + } + if(doWrite || ((val >> 16) & 0xff)) { + r = tlvbufAddOctet(gf, (val >> 16) & 0xff), doWrite = 1; + if(r != 0) goto done; + } + if(doWrite || ((val >> 8) & 0xff)) { + r = tlvbufAddOctet(gf, (val >> 8) & 0xff), doWrite = 1; + if(r != 0) goto done; + } + r = tlvbufAddOctet(gf, val & 0xff); +done: return r; +} + + +int +tlv8Write(gtfile gf, int flags, int tlvtype, int len) +{ + int r; + r = tlvbufAddOctet(gf, (flags << 5)|tlvtype); + if(r != 0) goto done; + r = tlvbufAddOctet(gf, len & 0xff); +done: return r; +} + +int +tlv16Write(gtfile gf, int flags, int tlvtype, uint16_t len) +{ + uint16_t typ; + int r; + typ = ((flags|1) << 15)|tlvtype; + r = tlvbufAddOctet(gf, typ >> 8); + if(r != 0) goto done; + r = tlvbufAddOctet(gf, typ & 0xff); + if(r != 0) goto done; + r = tlvbufAddOctet(gf, (len >> 8) & 0xff); + if(r != 0) goto done; + r = tlvbufAddOctet(gf, len & 0xff); +done: return r; +} + +int +tlvFlush(gtfile gf) +{ + return (gf->tlvIdx == 0) ? 0 : tlvbufPhysWrite(gf); +} + +int +tlvWriteHash(gtfile gf, uint16_t tlvtype, GTDataHash *rec) +{ + unsigned tlvlen; + int r; + tlvlen = 1 + rec->digest_length; + r = tlv16Write(gf, 0x00, tlvtype, tlvlen); + if(r != 0) goto done; + r = tlvbufAddOctet(gf, hashIdentifier(gf->hashAlg)); + if(r != 0) goto done; + r = tlvbufAddOctetString(gf, rec->digest, rec->digest_length); +done: return r; +} + +int +tlvWriteBlockSig(gtfile gf, uchar *der, uint16_t lenDer) +{ + unsigned tlvlen; + uint8_t tlvlenRecords; + int r; + + tlvlenRecords = tlvbufGetInt64OctetSize(gf->nRecords); + tlvlen = 2 + 1 /* hash algo TLV */ + + 2 + hashOutputLengthOctets(gf->hashAlg) /* iv */ + + 2 + 1 + gf->lenBlkStrtHash /* last hash */ + + 2 + tlvlenRecords /* rec-count */ + + 4 + lenDer /* rfc-3161 */; + /* write top-level TLV object (block-sig */ + r = tlv16Write(gf, 0x00, 0x0902, tlvlen); + if(r != 0) goto done; + /* and now write the children */ + //FIXME: flags??? + /* hash-algo */ + r = tlv8Write(gf, 0x00, 0x00, 1); + if(r != 0) goto done; + r = tlvbufAddOctet(gf, hashIdentifier(gf->hashAlg)); + if(r != 0) goto done; + /* block-iv */ + r = tlv8Write(gf, 0x00, 0x01, hashOutputLengthOctets(gf->hashAlg)); + if(r != 0) goto done; + r = tlvbufAddOctetString(gf, gf->IV, hashOutputLengthOctets(gf->hashAlg)); + if(r != 0) goto done; + /* last-hash */ + r = tlv8Write(gf, 0x00, 0x02, gf->lenBlkStrtHash+1); + if(r != 0) goto done; + r = tlvbufAddOctet(gf, hashIdentifier(gf->hashAlg)); + if(r != 0) goto done; + r = tlvbufAddOctetString(gf, gf->blkStrtHash, gf->lenBlkStrtHash); + if(r != 0) goto done; + /* rec-count */ + r = tlv8Write(gf, 0x00, 0x03, tlvlenRecords); + if(r != 0) goto done; + r = tlvbufAddInt64(gf, gf->nRecords); + if(r != 0) goto done; + /* rfc-3161 */ + r = tlv16Write(gf, 0x00, 0x906, lenDer); + if(r != 0) goto done; + r = tlvbufAddOctetString(gf, der, lenDer); +done: return r; +} + +/* support for old platforms - graceful degrade */ +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif +/* read rsyslog log state file; if we cannot access it or the + * contents looks invalid, we flag it as non-present (and thus + * begin a new hash chain). + * The context is initialized accordingly. + */ +static void +readStateFile(gtfile gf) +{ + int fd; + struct rsgtstatefile sf; + + fd = open((char*)gf->statefilename, O_RDONLY|O_NOCTTY|O_CLOEXEC, 0600); + if(fd == -1) goto err; + + if(read(fd, &sf, sizeof(sf)) != sizeof(sf)) goto err; + if(strncmp(sf.hdr, "GTSTAT10", 8)) goto err; + + gf->lenBlkStrtHash = sf.lenHash; + gf->blkStrtHash = calloc(1, gf->lenBlkStrtHash); + if(read(fd, gf->blkStrtHash, gf->lenBlkStrtHash) + != gf->lenBlkStrtHash) { + free(gf->blkStrtHash); + goto err; + } +return; + +err: + gf->lenBlkStrtHash = hashOutputLengthOctets(gf->hashAlg); + gf->blkStrtHash = calloc(1, gf->lenBlkStrtHash); +} + +/* persist all information that we need to re-open and append + * to a log signature file. + */ +static void +writeStateFile(gtfile gf) +{ + int fd; + struct rsgtstatefile sf; + + fd = open((char*)gf->statefilename, + O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY|O_CLOEXEC, 0600); + if(fd == -1) + goto done; + + memcpy(sf.hdr, "GTSTAT10", 8); + sf.hashID = hashIdentifier(gf->hashAlg); + sf.lenHash = gf->x_prev->len; + /* if the write fails, we cannot do anything against that. We check + * the condition just to keep the compiler happy. + */ + if(write(fd, &sf, sizeof(sf))){}; + if(write(fd, gf->x_prev->data, gf->x_prev->len)){}; + close(fd); +done: return; +} + + +int +tlvClose(gtfile gf) +{ + int r; + r = tlvFlush(gf); + close(gf->fd); + gf->fd = -1; + writeStateFile(gf); + return r; +} + + +/* note: if file exists, the last hash for chaining must + * be read from file. + */ +int +tlvOpen(gtfile gf, char *hdr, unsigned lenHdr) +{ + int r = 0; + gf->fd = open((char*)gf->sigfilename, + O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC, 0600); + if(gf->fd == -1) { + /* looks like we need to create a new file */ + gf->fd = open((char*)gf->sigfilename, + O_WRONLY|O_CREAT|O_NOCTTY|O_CLOEXEC, 0600); + if(gf->fd == -1) { + r = RSGTE_IO; + goto done; + } + memcpy(gf->tlvBuf, hdr, lenHdr); + gf->tlvIdx = lenHdr; + } else { + gf->tlvIdx = 0; /* header already present! */ + } + /* we now need to obtain the last previous hash, so that + * we can continue the hash chain. We do not check for error + * as a state file error can be recovered by graceful degredation. + */ + readStateFile(gf); +done: return r; +} + +/* + * As of some Linux and security expert I spoke to, /dev/urandom + * provides very strong random numbers, even if it runs out of + * entropy. As far as he knew, this is save for all applications + * (and he had good proof that I currently am not permitted to + * reproduce). -- rgerhards, 2013-03-04 + */ +void +seedIV(gtfile gf) +{ + int hashlen; + int fd; + + hashlen = hashOutputLengthOctets(gf->hashAlg); + gf->IV = malloc(hashlen); /* do NOT zero-out! */ + /* if we cannot obtain data from /dev/urandom, we use whatever + * is present at the current memory location as random data. Of + * course, this is very weak and we should consider a different + * option, especially when not running under Linux (for Linux, + * unavailability of /dev/urandom is just a theoretic thing, it + * will always work...). -- TODO -- rgerhards, 2013-03-06 + */ + if((fd = open("/dev/urandom", O_RDONLY)) > 0) { + if(read(fd, gf->IV, hashlen)) {}; /* keep compiler happy */ + close(fd); + } +} + +gtctx +rsgtCtxNew(void) +{ + gtctx ctx; + ctx = calloc(1, sizeof(struct gtctx_s)); + ctx->hashAlg = GT_HASHALG_SHA256; + ctx->errFunc = NULL; + ctx->usrptr = NULL; + ctx->timestamper = strdup( + "http://stamper.guardtime.net/gt-signingservice"); + return ctx; +} + +/* either returns gtfile object or NULL if something went wrong */ +gtfile +rsgtCtxOpenFile(gtctx ctx, unsigned char *logfn) +{ + gtfile gf; + char fn[MAXFNAME+1]; + + if((gf = rsgtfileConstruct(ctx)) == NULL) + goto done; + + snprintf(fn, sizeof(fn), "%s.gtsig", logfn); + fn[MAXFNAME] = '\0'; /* be on save side */ + gf->sigfilename = (uchar*) strdup(fn); + snprintf(fn, sizeof(fn), "%s.gtstate", logfn); + fn[MAXFNAME] = '\0'; /* be on save side */ + gf->statefilename = (uchar*) strdup(fn); + if(tlvOpen(gf, LOGSIGHDR, sizeof(LOGSIGHDR)-1) != 0) { + reportErr(ctx, "signature file open failed"); + gf = NULL; + } +done: return gf; +} + + +/* returns 0 on succes, 1 if algo is unknown */ +int +rsgtSetHashFunction(gtctx ctx, char *algName) +{ + int r = 0; + if(!strcmp(algName, "SHA2-256")) + ctx->hashAlg = GT_HASHALG_SHA256; + else if(!strcmp(algName, "SHA2-384")) + ctx->hashAlg = GT_HASHALG_SHA384; + else if(!strcmp(algName, "SHA2-512")) + ctx->hashAlg = GT_HASHALG_SHA512; + else if(!strcmp(algName, "SHA1")) + ctx->hashAlg = GT_HASHALG_SHA1; + else if(!strcmp(algName, "RIPEMD-160")) + ctx->hashAlg = GT_HASHALG_RIPEMD160; + else if(!strcmp(algName, "SHA2-224")) + ctx->hashAlg = GT_HASHALG_SHA224; + else + r = 1; + return r; +} + +int +rsgtfileDestruct(gtfile gf) +{ + int r = 0; + if(gf == NULL) + goto done; + + if(!gf->disabled && gf->bInBlk) { + r = sigblkFinish(gf); + if(r != 0) gf->disabled = 1; + } + if(!gf->disabled) + r = tlvClose(gf); + free(gf->sigfilename); + free(gf->statefilename); + free(gf->IV); + free(gf->blkStrtHash); + rsgtimprintDel(gf->x_prev); + free(gf); +done: return r; +} + +void +rsgtCtxDel(gtctx ctx) +{ + if(ctx != NULL) { + free(ctx->timestamper); + free(ctx); + } +} + +/* new sigblk is initialized, but maybe in existing ctx */ +void +sigblkInit(gtfile gf) +{ + if(gf == NULL) goto done; + seedIV(gf); + memset(gf->roots_valid, 0, sizeof(gf->roots_valid)/sizeof(char)); + gf->nRoots = 0; + gf->nRecords = 0; + gf->bInBlk = 1; +done: return; +} + + +/* concat: add IV to buffer */ +static inline void +bufAddIV(gtfile gf, uchar *buf, size_t *len) +{ + memcpy(buf+*len, gf->IV, hashOutputLengthOctets(gf->hashAlg)); + *len += sizeof(gf->IV); +} + + +/* concat: add imprint to buffer */ +static inline void +bufAddImprint(gtfile gf, uchar *buf, size_t *len, imprint_t *imp) +{ + if(imp == NULL) { + /* TODO: how to get the REAL HASH ID? --> add field? */ + buf[*len] = hashIdentifier(gf->hashAlg); + ++(*len); + memcpy(buf+*len, gf->blkStrtHash, gf->lenBlkStrtHash); + *len += gf->lenBlkStrtHash; + } else { + buf[*len] = imp->hashID; + ++(*len); + memcpy(buf+*len, imp->data, imp->len); + *len += imp->len; + } +} +/* concat: add hash to buffer */ +static inline void +bufAddHash(gtfile gf, uchar *buf, size_t *len, GTDataHash *hash) +{ + buf[*len] = hashIdentifier(gf->hashAlg); + ++(*len); + memcpy(buf+*len, hash->digest, hash->digest_length); + *len += hash->digest_length; +} +/* concat: add tree level to buffer */ +static inline void +bufAddLevel(uchar *buf, size_t *len, uint8_t level) +{ + memcpy(buf+*len, &level, sizeof(level)); + *len += sizeof(level); +} + + +int +hash_m(gtfile gf, GTDataHash **m) +{ + int rgt; + uchar concatBuf[16*1024]; + size_t len = 0; + int r = 0; + + bufAddImprint(gf, concatBuf, &len, gf->x_prev); + bufAddIV(gf, concatBuf, &len); + rgt = GTDataHash_create(gf->hashAlg, concatBuf, len, m); + if(rgt != GT_OK) { + reportGTAPIErr(gf->ctx, gf, "GTDataHash_create", rgt); + r = RSGTE_HASH_CREATE; + goto done; + } +done: return r; +} + +int +hash_r(gtfile gf, GTDataHash **r, const uchar *rec, const size_t len) +{ + int ret = 0, rgt; + rgt = GTDataHash_create(gf->hashAlg, rec, len, r); + if(rgt != GT_OK) { + reportGTAPIErr(gf->ctx, gf, "GTDataHash_create", rgt); + ret = RSGTE_HASH_CREATE; + goto done; + } +done: return ret; +} + + +int +hash_node(gtfile gf, GTDataHash **node, GTDataHash *m, GTDataHash *rec, + uint8_t level) +{ + int r = 0, rgt; + uchar concatBuf[16*1024]; + size_t len = 0; + + bufAddHash(gf, concatBuf, &len, m); + bufAddHash(gf, concatBuf, &len, rec); + bufAddLevel(concatBuf, &len, level); + rgt = GTDataHash_create(gf->hashAlg, concatBuf, len, node); + if(rgt != GT_OK) { + reportGTAPIErr(gf->ctx, gf, "GTDataHash_create", rgt); + r = RSGTE_HASH_CREATE; + goto done; + } +done: return r; +} + + +int +sigblkAddRecord(gtfile gf, const uchar *rec, const size_t len) +{ + GTDataHash *x; /* current hash */ + GTDataHash *m, *r, *t, *t_del; + uint8_t j; + int ret = 0; + + if(gf == NULL || gf->disabled) goto done; + if((ret = hash_m(gf, &m)) != 0) goto done; + if((ret = hash_r(gf, &r, rec, len)) != 0) goto done; + if(gf->bKeepRecordHashes) + tlvWriteHash(gf, 0x0900, r); + if((ret = hash_node(gf, &x, m, r, 1)) != 0) goto done; /* hash leaf */ + /* persists x here if Merkle tree needs to be persisted! */ + if(gf->bKeepTreeHashes) + tlvWriteHash(gf, 0x0901, x); + rsgtimprintDel(gf->x_prev); + gf->x_prev = rsgtImprintFromGTDataHash(x); + /* add x to the forest as new leaf, update roots list */ + t = x; + for(j = 0 ; j < gf->nRoots ; ++j) { + if(gf->roots_valid[j] == 0) { + gf->roots_hash[j] = t; + gf->roots_valid[j] = 1; + t = NULL; + break; + } else if(t != NULL) { + /* hash interim node */ + t_del = t; + ret = hash_node(gf, &t, gf->roots_hash[j], t_del, j+2); + gf->roots_valid[j] = 0; + GTDataHash_free(gf->roots_hash[j]); + GTDataHash_free(t_del); + if(ret != 0) goto done; + if(gf->bKeepTreeHashes) + tlvWriteHash(gf, 0x0901, t); + } + } + if(t != NULL) { + /* new level, append "at the top" */ + gf->roots_hash[gf->nRoots] = t; + gf->roots_valid[gf->nRoots] = 1; + ++gf->nRoots; + assert(gf->nRoots < MAX_ROOTS); + t = NULL; + } + ++gf->nRecords; + + /* cleanup (x is cleared as part of the roots array) */ + GTDataHash_free(m); + GTDataHash_free(r); + + if(gf->nRecords == gf->blockSizeLimit) { + ret = sigblkFinish(gf); + if(ret != 0) goto done; + sigblkInit(gf); + } +done: + if(ret != 0) { + gf->disabled = 1; + } + return ret; +} + +static int +timestampIt(gtfile gf, GTDataHash *hash) +{ + unsigned char *der = NULL; + size_t lenDer; + int r = GT_OK; + int ret = 0; + GTTimestamp *timestamp = NULL; + + /* Get the timestamp. */ + r = GTHTTP_createTimestampHash(hash, gf->ctx->timestamper, ×tamp); + + if(r != GT_OK) { + reportGTAPIErr(gf->ctx, gf, "GTHTTP_createTimestampHash", r); + ret = 1; + goto done; + } + + /* Encode timestamp. */ + r = GTTimestamp_getDEREncoded(timestamp, &der, &lenDer); + if(r != GT_OK) { + reportGTAPIErr(gf->ctx, gf, "GTTimestamp_getDEREncoded", r); + ret = 1; + goto done; + } + + tlvWriteBlockSig(gf, der, lenDer); + +done: + GT_free(der); + GTTimestamp_free(timestamp); + return ret; +} + + +int +sigblkFinish(gtfile gf) +{ + GTDataHash *root, *rootDel; + int8_t j; + int ret = 0; + + if(gf->nRecords == 0) + goto done; + + root = NULL; + for(j = 0 ; j < gf->nRoots ; ++j) { + if(root == NULL) { + root = gf->roots_valid[j] ? gf->roots_hash[j] : NULL; + gf->roots_valid[j] = 0; + } else if(gf->roots_valid[j]) { + rootDel = root; + ret = hash_node(gf, &root, gf->roots_hash[j], rootDel, j+2); + gf->roots_valid[j] = 0; + GTDataHash_free(gf->roots_hash[j]); + GTDataHash_free(rootDel); + if(ret != 0) goto done; /* checks hash_node() result! */ + } + } + if((ret = timestampIt(gf, root)) != 0) goto done; + + GTDataHash_free(root); + free(gf->blkStrtHash); + gf->lenBlkStrtHash = gf->x_prev->len; + gf->blkStrtHash = malloc(gf->lenBlkStrtHash); + memcpy(gf->blkStrtHash, gf->x_prev->data, gf->x_prev->len); +done: + gf->bInBlk = 0; + return ret; +} diff --git a/runtime/librsgt.h b/runtime/librsgt.h new file mode 100644 index 00000000..bfcc4628 --- /dev/null +++ b/runtime/librsgt.h @@ -0,0 +1,388 @@ +/* librsgt.h - rsyslog's guardtime support library + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_LIBRSGT_H +#define INCLUDED_LIBRSGT_H +#include <gt_base.h> + +/* Max number of roots inside the forest. This permits blocks of up to + * 2^MAX_ROOTS records. We assume that 64 is sufficient for all use + * cases ;) [and 64 is not really a waste of memory, so we do not even + * try to work with reallocs and such...] + */ +#define MAX_ROOTS 64 +#define LOGSIGHDR "LOGSIG10" + +/* context for gt calls. This primarily serves as a container for the + * config settings. The actual file-specific data is kept in gtfile. + */ +struct gtctx_s { + enum GTHashAlgorithm hashAlg; + uint8_t bKeepRecordHashes; + uint8_t bKeepTreeHashes; + uint64_t blockSizeLimit; + char *timestamper; + void (*errFunc)(void *, unsigned char*); + void *usrptr; /* for error function */ +}; +typedef struct gtctx_s *gtctx; +typedef struct gtfile_s *gtfile; +typedef struct gterrctx_s gterrctx_t; +typedef struct imprint_s imprint_t; +typedef struct block_sig_s block_sig_t; +typedef struct tlvrecord_s tlvrecord_t; + +/* this describes a file, as far as librsgt is concerned */ +struct gtfile_s { + /* the following data items are mirrored from gtctx to + * increase cache hit ratio (they are frequently accesed). + */ + enum GTHashAlgorithm hashAlg; + uint8_t bKeepRecordHashes; + uint8_t bKeepTreeHashes; + /* end mirrored properties */ + uint8_t disabled; /* permits to disable this file --> set to 1 */ + uint64_t blockSizeLimit; + uint8_t *IV; /* initial value for blinding masks */ + imprint_t *x_prev; /* last leaf hash (maybe of previous block) --> preserve on term */ + unsigned char *sigfilename; + unsigned char *statefilename; + int fd; + unsigned char *blkStrtHash; /* last hash from previous block */ + uint16_t lenBlkStrtHash; + uint64_t nRecords; /* current number of records in current block */ + uint64_t bInBlk; /* are we currently inside a blk --> need to finish on close */ + int8_t nRoots; + /* algo engineering: roots structure is split into two arrays + * in order to improve cache hits. + */ + int8_t roots_valid[MAX_ROOTS]; + GTDataHash *roots_hash[MAX_ROOTS]; + /* data members for the associated TLV file */ + char tlvBuf[4096]; + int tlvIdx; /* current index into tlvBuf */ + gtctx ctx; +}; + +struct tlvrecord_s { + uint16_t tlvtype; + uint16_t tlvlen; + uint8_t hdr[4]; /* the raw header (as persisted to file) */ + uint8_t lenHdr; /* length of raw header */ + uint8_t data[64*1024]; /* the actual data part (of length tlvlen) */ +}; + +/* The following structure describes the "error context" to be used + * for verification and similiar reader functions. While verifying, + * we need some information (like filenames or block numbers) that + * is not readily available from the other objects (or not even known + * to librsgt). In order to provide meaningful error messages, this + * information must be passed in from the external callers. In order + * to centralize information (and make it more manageable), we use + * ths error context here, which contains everything needed to + * generate good error messages. Members of this structure are + * maintained both by library users (the callers) as well as + * the library itself. Who does what simply depends on who has + * the relevant information. + */ +struct gterrctx_s { + FILE *fp; /**< file for error messages */ + char *filename; + uint8_t verbose; + uint64_t recNumInFile; + uint64_t recNum; + uint64_t blkNum; + uint8_t treeLevel; + GTDataHash *computedHash; + GTDataHash *lefthash, *righthash; /* hashes to display if tree hash fails */ + imprint_t *fileHash; + int gtstate; /* status from last relevant GT.*() function call */ + char *errRec; + char *frstRecInBlk; /* This holds the first message seen inside the current block */ +}; + +struct imprint_s { + uint8_t hashID; + int len; + uint8_t *data; +}; + +#define SIGID_RFC3161 0 +struct block_sig_s { + uint8_t hashID; + uint8_t sigID; /* what type of *signature*? */ + uint8_t *iv; + imprint_t lastHash; + uint64_t recCount; + struct { + struct { + uint8_t *data; + size_t len; /* must be size_t due to GT API! */ + } der; + } sig; +}; + + +/* the following defines the gtstate file record. Currently, this record + * is fixed, we may change that over time. + */ +struct rsgtstatefile { + char hdr[8]; /* must be "GTSTAT10" */ + uint8_t hashID; + uint8_t lenHash; + /* after that, the hash value is contained within the file */ +}; + +/* Flags and record types for TLV handling */ +#define RSGT_FLAG_TLV16 0x20 + +/* error states */ +#define RSGTE_IO 1 /* any kind of io error */ +#define RSGTE_FMT 2 /* data fromat error */ +#define RSGTE_INVLTYP 3 /* invalid TLV type record (unexcpected at this point) */ +#define RSGTE_OOM 4 /* ran out of memory */ +#define RSGTE_LEN 5 /* error related to length records */ +#define RSGTE_TS_EXTEND 6/* error extending timestamp */ +#define RSGTE_INVLD_RECCNT 7/* mismatch between actual records and records + given in block-sig record */ +#define RSGTE_INVLHDR 8/* invalid file header */ +#define RSGTE_EOF 9 /* specific EOF */ +#define RSGTE_MISS_REC_HASH 10 /* record hash missing when expected */ +#define RSGTE_MISS_TREE_HASH 11 /* tree hash missing when expected */ +#define RSGTE_INVLD_REC_HASH 12 /* invalid record hash (failed verification) */ +#define RSGTE_INVLD_TREE_HASH 13 /* invalid tree hash (failed verification) */ +#define RSGTE_INVLD_REC_HASHID 14 /* invalid record hash ID (failed verification) */ +#define RSGTE_INVLD_TREE_HASHID 15 /* invalid tree hash ID (failed verification) */ +#define RSGTE_MISS_BLOCKSIG 16 /* block signature record missing when expected */ +#define RSGTE_INVLD_TIMESTAMP 17 /* RFC3161 timestamp is invalid */ +#define RSGTE_TS_DERDECODE 18 /* error DER-Decoding a timestamp */ +#define RSGTE_TS_DERENCODE 19 /* error DER-Encoding a timestamp */ +#define RSGTE_HASH_CREATE 20 /* error creating a hash */ + +/* the following function maps RSGTE_* state to a string - must be updated + * whenever a new state is added. + * Note: it is thread-safe to call this function, as it returns a pointer + * into constant memory pool. + */ +static inline char * +RSGTE2String(int err) +{ + switch(err) { + case 0: + return "success"; + case RSGTE_IO: + return "i/o error"; + case RSGTE_FMT: + return "data format error"; + case RSGTE_INVLTYP: + return "invalid/unexpected tlv record type"; + case RSGTE_OOM: + return "out of memory"; + case RSGTE_LEN: + return "length record problem"; + case RSGTE_TS_EXTEND: + return "error extending timestamp"; + case RSGTE_INVLD_RECCNT: + return "mismatch between actual record count and number in block signature record"; + case RSGTE_INVLHDR: + return "invalid file header"; + case RSGTE_EOF: + return "EOF"; + case RSGTE_MISS_REC_HASH: + return "record hash missing"; + case RSGTE_MISS_TREE_HASH: + return "tree hash missing"; + case RSGTE_INVLD_REC_HASH: + return "record hash mismatch"; + case RSGTE_INVLD_TREE_HASH: + return "tree hash mismatch"; + case RSGTE_INVLD_REC_HASHID: + return "invalid record hash ID"; + case RSGTE_INVLD_TREE_HASHID: + return "invalid tree hash ID"; + case RSGTE_MISS_BLOCKSIG: + return "missing block signature record"; + case RSGTE_INVLD_TIMESTAMP: + return "RFC3161 timestamp invalid"; + case RSGTE_TS_DERDECODE: + return "error DER-decoding RFC3161 timestamp"; + case RSGTE_TS_DERENCODE: + return "error DER-encoding RFC3161 timestamp"; + case RSGTE_HASH_CREATE: + return "error creating hash"; + default: + return "unknown error"; + } +} + + +static inline uint16_t +hashOutputLengthOctets(uint8_t hashID) +{ + switch(hashID) { + case GT_HASHALG_SHA1: /* paper: SHA1 */ + return 20; + case GT_HASHALG_RIPEMD160: /* paper: RIPEMD-160 */ + return 20; + case GT_HASHALG_SHA224: /* paper: SHA2-224 */ + return 28; + case GT_HASHALG_SHA256: /* paper: SHA2-256 */ + return 32; + case GT_HASHALG_SHA384: /* paper: SHA2-384 */ + return 48; + case GT_HASHALG_SHA512: /* paper: SHA2-512 */ + return 64; + default:return 32; + } +} + +static inline uint8_t +hashIdentifier(enum GTHashAlgorithm hashID) +{ + switch(hashID) { + case GT_HASHALG_SHA1: /* paper: SHA1 */ + return 0x00; + case GT_HASHALG_RIPEMD160: /* paper: RIPEMD-160 */ + return 0x02; + case GT_HASHALG_SHA224: /* paper: SHA2-224 */ + return 0x03; + case GT_HASHALG_SHA256: /* paper: SHA2-256 */ + return 0x01; + case GT_HASHALG_SHA384: /* paper: SHA2-384 */ + return 0x04; + case GT_HASHALG_SHA512: /* paper: SHA2-512 */ + return 0x05; + default:return 0xff; + } +} +static inline char * +hashAlgName(uint8_t hashID) +{ + switch(hashID) { + case GT_HASHALG_SHA1: + return "SHA1"; + case GT_HASHALG_RIPEMD160: + return "RIPEMD-160"; + case GT_HASHALG_SHA224: + return "SHA2-224"; + case GT_HASHALG_SHA256: + return "SHA2-256"; + case GT_HASHALG_SHA384: + return "SHA2-384"; + case GT_HASHALG_SHA512: + return "SHA2-512"; + default:return "[unknown]"; + } +} +static inline enum GTHashAlgorithm +hashID2Alg(uint8_t hashID) +{ + switch(hashID) { + case 0x00: + return GT_HASHALG_SHA1; + case 0x02: + return GT_HASHALG_RIPEMD160; + case 0x03: + return GT_HASHALG_SHA224; + case 0x01: + return GT_HASHALG_SHA256; + case 0x04: + return GT_HASHALG_SHA384; + case 0x05: + return GT_HASHALG_SHA512; + default: + return 0xff; + } +} +static inline char * +sigTypeName(uint8_t sigID) +{ + switch(sigID) { + case SIGID_RFC3161: + return "RFC3161"; + default:return "[unknown]"; + } +} +static inline uint16_t +getIVLen(block_sig_t *bs) +{ + return hashOutputLengthOctets(bs->hashID); +} +static inline void +rsgtSetTimestamper(gtctx ctx, char *timestamper) +{ + free(ctx->timestamper); + ctx->timestamper = strdup(timestamper); +} +static inline void +rsgtSetBlockSizeLimit(gtctx ctx, uint64_t limit) +{ + ctx->blockSizeLimit = limit; +} +static inline void +rsgtSetKeepRecordHashes(gtctx ctx, int val) +{ + ctx->bKeepRecordHashes = val; +} +static inline void +rsgtSetKeepTreeHashes(gtctx ctx, int val) +{ + ctx->bKeepTreeHashes = val; +} + +int rsgtSetHashFunction(gtctx ctx, char *algName); +int rsgtInit(char *usragent); +void rsgtExit(void); +gtctx rsgtCtxNew(void); +void rsgtsetErrFunc(gtctx ctx, void (*func)(void*, unsigned char *), void *usrptr); +gtfile rsgtCtxOpenFile(gtctx ctx, unsigned char *logfn); +int rsgtfileDestruct(gtfile gf); +void rsgtCtxDel(gtctx ctx); +void sigblkInit(gtfile gf); +int sigblkAddRecord(gtfile gf, const unsigned char *rec, const size_t len); +int sigblkFinish(gtfile gf); +imprint_t * rsgtImprintFromGTDataHash(GTDataHash *hash); +void rsgtimprintDel(imprint_t *imp); +/* reader functions */ +int rsgt_tlvrdHeader(FILE *fp, unsigned char *hdr); +int rsgt_tlvrd(FILE *fp, tlvrecord_t *rec, void *obj); +void rsgt_tlvprint(FILE *fp, uint16_t tlvtype, void *obj, uint8_t verbose); +void rsgt_printBLOCK_SIG(FILE *fp, block_sig_t *bs, uint8_t verbose); +int rsgt_getBlockParams(FILE *fp, uint8_t bRewind, block_sig_t **bs, uint8_t *bHasRecHashes, uint8_t *bHasIntermedHashes); +int rsgt_chkFileHdr(FILE *fp, char *expect); +gtfile rsgt_vrfyConstruct_gf(void); +void rsgt_vrfyBlkInit(gtfile gf, block_sig_t *bs, uint8_t bHasRecHashes, uint8_t bHasIntermedHashes); +int rsgt_vrfy_nextRec(block_sig_t *bs, gtfile gf, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, gterrctx_t *ectx); +int verifyBLOCK_SIG(block_sig_t *bs, gtfile gf, FILE *sigfp, FILE *nsigfp, uint8_t bExtend, gterrctx_t *ectx); +void rsgt_errctxInit(gterrctx_t *ectx); +void rsgt_errctxExit(gterrctx_t *ectx); +void rsgt_errctxSetErrRec(gterrctx_t *ectx, char *rec); +void rsgt_errctxFrstRecInBlk(gterrctx_t *ectx, char *rec); +void rsgt_objfree(uint16_t tlvtype, void *obj); + + +/* TODO: replace these? */ +int hash_m(gtfile gf, GTDataHash **m); +int hash_r(gtfile gf, GTDataHash **r, const unsigned char *rec, const size_t len); +int hash_node(gtfile gf, GTDataHash **node, GTDataHash *m, GTDataHash *r, uint8_t level); +extern char *rsgt_read_puburl; /**< url of publication server */ +extern uint8_t rsgt_read_showVerified; + +#endif /* #ifndef INCLUDED_LIBRSGT_H */ diff --git a/runtime/librsgt_read.c b/runtime/librsgt_read.c new file mode 100644 index 00000000..a6e33160 --- /dev/null +++ b/runtime/librsgt_read.c @@ -0,0 +1,1092 @@ +/* librsgt_read.c - rsyslog's guardtime support library + * This includes functions used for reading signature (and + * other related) files. Well, actually it also contains + * some writing functionality, but only as far as rsyslog + * itself is not concerned, but "just" the utility programs. + * + * This part of the library uses C stdio and expects that the + * caller will open and close the file to be read itself. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include <stdio.h> +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <stdint.h> +#include <assert.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <gt_http.h> + +#include "librsgt.h" + +typedef unsigned char uchar; +#ifndef VERSION +#define VERSION "no-version" +#endif +#define MAXFNAME 1024 + +static int rsgt_read_debug = 0; +char *rsgt_read_puburl = "http://verify.guardtime.com/gt-controlpublications.bin"; +char *rsgt_extend_puburl = "http://verifier.guardtime.net/gt-extendingservice"; +uint8_t rsgt_read_showVerified = 0; + +/* macro to obtain next char from file including error tracking */ +#define NEXTC if((c = fgetc(fp)) == EOF) { \ + r = feof(fp) ? RSGTE_EOF : RSGTE_IO; \ + goto done; \ + } + +/* check return state of operation and abort, if non-OK */ +#define CHKr(code) if((r = code) != 0) goto done + + +/* if verbose==0, only the first and last two octets are shown, + * otherwise everything. + */ +static void +outputHexBlob(FILE *fp, uint8_t *blob, uint16_t len, uint8_t verbose) +{ + unsigned i; + if(verbose || len <= 8) { + for(i = 0 ; i < len ; ++i) + fprintf(fp, "%2.2x", blob[i]); + } else { + fprintf(fp, "%2.2x%2.2x%2.2x[...]%2.2x%2.2x%2.2x", + blob[0], blob[1], blob[2], + blob[len-3], blob[len-2], blob[len-1]); + } +} + +static inline void +outputHash(FILE *fp, char *hdr, uint8_t *data, uint16_t len, uint8_t verbose) +{ + fprintf(fp, "%s", hdr); + outputHexBlob(fp, data, len, verbose); + fputc('\n', fp); +} + +void +rsgt_errctxInit(gterrctx_t *ectx) +{ + ectx->fp = NULL; + ectx->filename = NULL; + ectx->recNum = 0; + ectx->gtstate = 0; + ectx->recNumInFile = 0; + ectx->blkNum = 0; + ectx->verbose = 0; + ectx->errRec = NULL; + ectx->frstRecInBlk = NULL; + ectx->fileHash = NULL; + ectx->lefthash = ectx->righthash = ectx->computedHash = NULL; +} +void +rsgt_errctxExit(gterrctx_t *ectx) +{ + free(ectx->filename); + free(ectx->frstRecInBlk); +} + +/* note: we do not copy the record, so the caller MUST not destruct + * it before processing of the record is completed. To remove the + * current record without setting a new one, call this function + * with rec==NULL. + */ +void +rsgt_errctxSetErrRec(gterrctx_t *ectx, char *rec) +{ + ectx->errRec = strdup(rec); +} +/* This stores the block's first record. Here we copy the data, + * as the caller will usually not preserve it long enough. + */ +void +rsgt_errctxFrstRecInBlk(gterrctx_t *ectx, char *rec) +{ + free(ectx->frstRecInBlk); + ectx->frstRecInBlk = strdup(rec); +} + +static void +reportError(int errcode, gterrctx_t *ectx) +{ + if(ectx->fp != NULL) { + fprintf(ectx->fp, "%s[%llu:%llu:%llu]: error[%u]: %s\n", + ectx->filename, + (long long unsigned) ectx->blkNum, (long long unsigned) ectx->recNum, + (long long unsigned) ectx->recNumInFile, + errcode, RSGTE2String(errcode)); + if(ectx->frstRecInBlk != NULL) + fprintf(ectx->fp, "\tBlock Start Record.: '%s'\n", ectx->frstRecInBlk); + if(ectx->errRec != NULL) + fprintf(ectx->fp, "\tRecord in Question.: '%s'\n", ectx->errRec); + if(ectx->computedHash != NULL) { + outputHash(ectx->fp, "\tComputed Hash......: ", ectx->computedHash->digest, + ectx->computedHash->digest_length, ectx->verbose); + } + if(ectx->fileHash != NULL) { + outputHash(ectx->fp, "\tSignature File Hash: ", ectx->fileHash->data, + ectx->fileHash->len, ectx->verbose); + } + if(errcode == RSGTE_INVLD_TREE_HASH || + errcode == RSGTE_INVLD_TREE_HASHID) { + fprintf(ectx->fp, "\tTree Level.........: %d\n", (int) ectx->treeLevel); + outputHash(ectx->fp, "\tTree Left Hash.....: ", ectx->lefthash->digest, + ectx->lefthash->digest_length, ectx->verbose); + outputHash(ectx->fp, "\tTree Right Hash....: ", ectx->righthash->digest, + ectx->righthash->digest_length, ectx->verbose); + } + if(errcode == RSGTE_INVLD_TIMESTAMP || + errcode == RSGTE_TS_DERDECODE) { + fprintf(ectx->fp, "\tPublication Server.: %s\n", rsgt_read_puburl); + fprintf(ectx->fp, "\tGT Verify Timestamp: [%u]%s\n", + ectx->gtstate, GTHTTP_getErrorString(ectx->gtstate)); + } + if(errcode == RSGTE_TS_EXTEND || + errcode == RSGTE_TS_DERDECODE) { + fprintf(ectx->fp, "\tExtending Server...: %s\n", rsgt_extend_puburl); + fprintf(ectx->fp, "\tGT Extend Timestamp: [%u]%s\n", + ectx->gtstate, GTHTTP_getErrorString(ectx->gtstate)); + } + if(errcode == RSGTE_TS_DERENCODE) { + fprintf(ectx->fp, "\tAPI return state...: [%u]%s\n", + ectx->gtstate, GTHTTP_getErrorString(ectx->gtstate)); + } + } +} + +/* obviously, this is not an error-reporting function. We still use + * ectx, as it has most information we need. + */ +static void +reportVerifySuccess(gterrctx_t *ectx, GTVerificationInfo *vrfyInf) +{ + if(ectx->fp != NULL) { + fprintf(ectx->fp, "%s[%llu:%llu:%llu]: block signature successfully verified\n", + ectx->filename, + (long long unsigned) ectx->blkNum, (long long unsigned) ectx->recNum, + (long long unsigned) ectx->recNumInFile); + if(ectx->frstRecInBlk != NULL) + fprintf(ectx->fp, "\tBlock Start Record.: '%s'\n", ectx->frstRecInBlk); + if(ectx->errRec != NULL) + fprintf(ectx->fp, "\tBlock End Record...: '%s'\n", ectx->errRec); + fprintf(ectx->fp, "\tGT Verify Timestamp: [%u]%s\n", + ectx->gtstate, GTHTTP_getErrorString(ectx->gtstate)); + GTVerificationInfo_print(ectx->fp, 0, vrfyInf); + } +} + +/** + * Write the provided record to the current file position. + * + * @param[in] fp file pointer for writing + * @param[out] rec tlvrecord to write + * + * @returns 0 if ok, something else otherwise + */ +static int +rsgt_tlvwrite(FILE *fp, tlvrecord_t *rec) +{ + int r = RSGTE_IO; + if(fwrite(rec->hdr, (size_t) rec->lenHdr, 1, fp) != 1) goto done; + if(fwrite(rec->data, (size_t) rec->tlvlen, 1, fp) != 1) goto done; + r = 0; +done: return r; +} + +/** + * Read a header from a binary file. + * @param[in] fp file pointer for processing + * @param[in] hdr buffer for the header. Must be 9 bytes + * (8 for header + NUL byte) + * @returns 0 if ok, something else otherwise + */ +int +rsgt_tlvrdHeader(FILE *fp, uchar *hdr) +{ + int r; + if(fread(hdr, 8, 1, fp) != 1) { + r = RSGTE_IO; + goto done; + } + hdr[8] = '\0'; + r = 0; +done: return r; +} + +/* read type a complete tlv record + */ +static int +rsgt_tlvRecRead(FILE *fp, tlvrecord_t *rec) +{ + int r = 1; + int c; + + NEXTC; + rec->hdr[0] = c; + rec->tlvtype = c & 0x1f; + if(c & 0x80) { /* tlv16? */ + rec->lenHdr = 4; + NEXTC; + rec->hdr[1] = c; + rec->tlvtype = (rec->tlvtype << 8) | c; + NEXTC; + rec->hdr[2] = c; + rec->tlvlen = c << 8; + NEXTC; + rec->hdr[3] = c; + rec->tlvlen |= c; + } else { + NEXTC; + rec->lenHdr = 2; + rec->hdr[1] = c; + rec->tlvlen = c; + } + if(fread(rec->data, (size_t) rec->tlvlen, 1, fp) != 1) { + r = RSGTE_IO; + goto done; + } + + if(rsgt_read_debug) + printf("read tlvtype %4.4x, len %u\n", (unsigned) rec->tlvtype, + (unsigned) rec->tlvlen); + r = 0; +done: return r; +} + +/* decode a sub-tlv record from an existing record's memory buffer + */ +static int +rsgt_tlvDecodeSUBREC(tlvrecord_t *rec, uint16_t *stridx, tlvrecord_t *newrec) +{ + int r = 1; + int c; + + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + c = rec->data[(*stridx)++]; + newrec->hdr[0] = c; + newrec->tlvtype = c & 0x1f; + if(c & 0x80) { /* tlv16? */ + newrec->lenHdr = 4; + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + c = rec->data[(*stridx)++]; + newrec->hdr[1] = c; + newrec->tlvtype = (newrec->tlvtype << 8) | c; + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + c = rec->data[(*stridx)++]; + newrec->hdr[2] = c; + newrec->tlvlen = c << 8; + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + c = rec->data[(*stridx)++]; + newrec->hdr[3] = c; + newrec->tlvlen |= c; + } else { + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + c = rec->data[(*stridx)++]; + newrec->lenHdr = 2; + newrec->hdr[1] = c; + newrec->tlvlen = c; + } + if(rec->tlvlen < *stridx + newrec->tlvlen) {r=RSGTE_LEN; goto done;} + memcpy(newrec->data, (rec->data)+(*stridx), newrec->tlvlen); + *stridx += newrec->tlvlen; + + if(rsgt_read_debug) + printf("read sub-tlv: tlvtype %4.4x, len %u\n", + (unsigned) newrec->tlvtype, + (unsigned) newrec->tlvlen); + r = 0; +done: return r; +} + + +static int +rsgt_tlvDecodeIMPRINT(tlvrecord_t *rec, imprint_t **imprint) +{ + int r = 1; + imprint_t *imp; + + if((imp = calloc(1, sizeof(imprint_t))) == NULL) { + r = RSGTE_OOM; + goto done; + } + + imp->hashID = rec->data[0]; + if(rec->tlvlen != 1 + hashOutputLengthOctets(imp->hashID)) { + r = RSGTE_LEN; + goto done; + } + imp->len = rec->tlvlen - 1; + if((imp->data = (uint8_t*)malloc(imp->len)) == NULL) {r=RSGTE_OOM;goto done;} + memcpy(imp->data, rec->data+1, imp->len); + *imprint = imp; + r = 0; +done: return r; +} + +static int +rsgt_tlvDecodeHASH_ALGO(tlvrecord_t *rec, uint16_t *strtidx, uint8_t *hashAlg) +{ + int r = 1; + tlvrecord_t subrec; + + CHKr(rsgt_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x00 && subrec.tlvlen == 1)) { + r = RSGTE_FMT; + goto done; + } + *hashAlg = subrec.data[0]; + r = 0; +done: return r; +} +static int +rsgt_tlvDecodeBLOCK_IV(tlvrecord_t *rec, uint16_t *strtidx, uint8_t **iv) +{ + int r = 1; + tlvrecord_t subrec; + + CHKr(rsgt_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x01)) { + r = RSGTE_INVLTYP; + goto done; + } + if((*iv = (uint8_t*)malloc(subrec.tlvlen)) == NULL) {r=RSGTE_OOM;goto done;} + memcpy(*iv, subrec.data, subrec.tlvlen); + r = 0; +done: return r; +} +static int +rsgt_tlvDecodeLAST_HASH(tlvrecord_t *rec, uint16_t *strtidx, imprint_t *imp) +{ + int r = 1; + tlvrecord_t subrec; + + CHKr(rsgt_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x02)) { r = RSGTE_INVLTYP; goto done; } + imp->hashID = subrec.data[0]; + if(subrec.tlvlen != 1 + hashOutputLengthOctets(imp->hashID)) { + r = RSGTE_LEN; + goto done; + } + imp->len = subrec.tlvlen - 1; + if((imp->data = (uint8_t*)malloc(imp->len)) == NULL) {r=RSGTE_OOM;goto done;} + memcpy(imp->data, subrec.data+1, subrec.tlvlen-1); + r = 0; +done: return r; +} +static int +rsgt_tlvDecodeREC_COUNT(tlvrecord_t *rec, uint16_t *strtidx, uint64_t *cnt) +{ + int r = 1; + int i; + uint64_t val; + tlvrecord_t subrec; + + CHKr(rsgt_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x03 && subrec.tlvlen <= 8)) { r = RSGTE_INVLTYP; goto done; } + val = 0; + for(i = 0 ; i < subrec.tlvlen ; ++i) { + val = (val << 8) + subrec.data[i]; + } + *cnt = val; + r = 0; +done: return r; +} +static int +rsgt_tlvDecodeSIG(tlvrecord_t *rec, uint16_t *strtidx, block_sig_t *bs) +{ + int r = 1; + tlvrecord_t subrec; + + CHKr(rsgt_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x0906)) { r = RSGTE_INVLTYP; goto done; } + bs->sig.der.len = subrec.tlvlen; + bs->sigID = SIGID_RFC3161; + if((bs->sig.der.data = (uint8_t*)malloc(bs->sig.der.len)) == NULL) {r=RSGTE_OOM;goto done;} + memcpy(bs->sig.der.data, subrec.data, bs->sig.der.len); + r = 0; +done: return r; +} + +static int +rsgt_tlvDecodeBLOCK_SIG(tlvrecord_t *rec, block_sig_t **blocksig) +{ + int r = 1; + uint16_t strtidx = 0; + block_sig_t *bs; + if((bs = calloc(1, sizeof(block_sig_t))) == NULL) { + r = RSGTE_OOM; + goto done; + } + CHKr(rsgt_tlvDecodeHASH_ALGO(rec, &strtidx, &(bs->hashID))); + CHKr(rsgt_tlvDecodeBLOCK_IV(rec, &strtidx, &(bs->iv))); + CHKr(rsgt_tlvDecodeLAST_HASH(rec, &strtidx, &(bs->lastHash))); + CHKr(rsgt_tlvDecodeREC_COUNT(rec, &strtidx, &(bs->recCount))); + CHKr(rsgt_tlvDecodeSIG(rec, &strtidx, bs)); + if(strtidx != rec->tlvlen) { + r = RSGTE_LEN; + goto done; + } + *blocksig = bs; + r = 0; +done: return r; +} +static int +rsgt_tlvRecDecode(tlvrecord_t *rec, void *obj) +{ + int r = 1; + switch(rec->tlvtype) { + case 0x0900: + case 0x0901: + r = rsgt_tlvDecodeIMPRINT(rec, obj); + if(r != 0) goto done; + break; + case 0x0902: + r = rsgt_tlvDecodeBLOCK_SIG(rec, obj); + if(r != 0) goto done; + break; + } +done: + return r; +} + +static int +rsgt_tlvrdRecHash(FILE *fp, FILE *outfp, imprint_t **imp) +{ + int r; + tlvrecord_t rec; + + if((r = rsgt_tlvrd(fp, &rec, imp)) != 0) goto done; + if(rec.tlvtype != 0x0900) { + r = RSGTE_MISS_REC_HASH; + rsgt_objfree(rec.tlvtype, *imp); + goto done; + } + if(outfp != NULL) + if((r = rsgt_tlvwrite(outfp, &rec)) != 0) goto done; + r = 0; +done: return r; +} + +static int +rsgt_tlvrdTreeHash(FILE *fp, FILE *outfp, imprint_t **imp) +{ + int r; + tlvrecord_t rec; + + if((r = rsgt_tlvrd(fp, &rec, imp)) != 0) goto done; + if(rec.tlvtype != 0x0901) { + r = RSGTE_MISS_TREE_HASH; + rsgt_objfree(rec.tlvtype, *imp); + goto done; + } + if(outfp != NULL) + if((r = rsgt_tlvwrite(outfp, &rec)) != 0) goto done; + r = 0; +done: return r; +} + +/* read BLOCK_SIG during verification phase */ +static int +rsgt_tlvrdVrfyBlockSig(FILE *fp, block_sig_t **bs, tlvrecord_t *rec) +{ + int r; + + if((r = rsgt_tlvrd(fp, rec, bs)) != 0) goto done; + if(rec->tlvtype != 0x0902) { + r = RSGTE_MISS_BLOCKSIG; + rsgt_objfree(rec->tlvtype, *bs); + goto done; + } + r = 0; +done: return r; +} + +/** + * Read the next "object" from file. This usually is + * a single TLV, but may be something larger, for + * example in case of a block-sig TLV record. + * Unknown type records are ignored (or run aborted + * if we are not permitted to skip). + * + * @param[in] fp file pointer for processing + * @param[out] tlvtype type of tlv record (top-level for + * structured objects. + * @param[out] tlvlen length of the tlv record value + * @param[out] obj pointer to object; This is a proper + * tlv record structure, which must be casted + * by the caller according to the reported type. + * The object must be freed by the caller (TODO: better way?) + * + * @returns 0 if ok, something else otherwise + */ +int +rsgt_tlvrd(FILE *fp, tlvrecord_t *rec, void *obj) +{ + int r; + if((r = rsgt_tlvRecRead(fp, rec)) != 0) goto done; + r = rsgt_tlvRecDecode(rec, obj); +done: return r; +} + + +/* return if a blob is all zero */ +static inline int +blobIsZero(uint8_t *blob, uint16_t len) +{ + int i; + for(i = 0 ; i < len ; ++i) + if(blob[i] != 0) + return 0; + return 1; +} + +static void +rsgt_printIMPRINT(FILE *fp, char *name, imprint_t *imp, uint8_t verbose) +{ + fprintf(fp, "%s", name); + outputHexBlob(fp, imp->data, imp->len, verbose); + fputc('\n', fp); +} + +static void +rsgt_printREC_HASH(FILE *fp, imprint_t *imp, uint8_t verbose) +{ + rsgt_printIMPRINT(fp, "[0x0900]Record hash: ", + imp, verbose); +} + +static void +rsgt_printINT_HASH(FILE *fp, imprint_t *imp, uint8_t verbose) +{ + rsgt_printIMPRINT(fp, "[0x0901]Tree hash..: ", + imp, verbose); +} + +/** + * Output a human-readable representation of a block_sig_t + * to proviced file pointer. This function is mainly inteded for + * debugging purposes or dumping tlv files. + * + * @param[in] fp file pointer to send output to + * @param[in] bsig ponter to block_sig_t to output + * @param[in] verbose if 0, abbreviate blob hexdump, else complete + */ +void +rsgt_printBLOCK_SIG(FILE *fp, block_sig_t *bs, uint8_t verbose) +{ + fprintf(fp, "[0x0902]Block Signature Record:\n"); + fprintf(fp, "\tPrevious Block Hash:\n"); + fprintf(fp, "\t Algorithm..: %s\n", hashAlgName(bs->lastHash.hashID)); + fprintf(fp, "\t Hash.......: "); + outputHexBlob(fp, bs->lastHash.data, bs->lastHash.len, verbose); + fputc('\n', fp); + if(blobIsZero(bs->lastHash.data, bs->lastHash.len)) + fprintf(fp, "\t NOTE: New Hash Chain Start!\n"); + fprintf(fp, "\tHash Algorithm: %s\n", hashAlgName(bs->hashID)); + fprintf(fp, "\tIV............: "); + outputHexBlob(fp, bs->iv, getIVLen(bs), verbose); + fputc('\n', fp); + fprintf(fp, "\tRecord Count..: %llu\n", bs->recCount); + fprintf(fp, "\tSignature Type: %s\n", sigTypeName(bs->sigID)); + fprintf(fp, "\tSignature Len.: %u\n", bs->sig.der.len); + fprintf(fp, "\tSignature.....: "); + outputHexBlob(fp, bs->sig.der.data, bs->sig.der.len, verbose); + fputc('\n', fp); +} + + +/** + * Output a human-readable representation of a tlv object. + * + * @param[in] fp file pointer to send output to + * @param[in] tlvtype type of tlv object (record) + * @param[in] verbose if 0, abbreviate blob hexdump, else complete + */ +void +rsgt_tlvprint(FILE *fp, uint16_t tlvtype, void *obj, uint8_t verbose) +{ + switch(tlvtype) { + case 0x0900: + rsgt_printREC_HASH(fp, obj, verbose); + break; + case 0x0901: + rsgt_printINT_HASH(fp, obj, verbose); + break; + case 0x0902: + rsgt_printBLOCK_SIG(fp, obj, verbose); + break; + default:fprintf(fp, "unknown tlv record %4.4x\n", tlvtype); + break; + } +} + +/** + * Free the provided object. + * + * @param[in] tlvtype type of tlv object (record) + * @param[in] obj the object to be destructed + */ +void +rsgt_objfree(uint16_t tlvtype, void *obj) +{ + switch(tlvtype) { + case 0x0900: + case 0x0901: + free(((imprint_t*)obj)->data); + break; + case 0x0902: + free(((block_sig_t*)obj)->iv); + free(((block_sig_t*)obj)->lastHash.data); + free(((block_sig_t*)obj)->sig.der.data); + break; + default:fprintf(stderr, "rsgt_objfree: unknown tlv record %4.4x\n", + tlvtype); + break; + } + free(obj); +} + +/** + * Read block parameters. This detects if the block contains the + * individual log hashes, the intermediate hashes and the overall + * block paramters (from the signature block). As we do not have any + * begin of block record, we do not know e.g. the hash algorithm or IV + * until reading the block signature record. And because the file is + * purely sequential and variable size, we need to read all records up to + * the next signature record. + * If a caller intends to verify a log file based on the parameters, + * he must re-read the file from the begining (we could keep things + * in memory, but this is impractical for large blocks). In order + * to facitate this, the function permits to rewind to the original + * read location when it is done. + * + * @param[in] fp file pointer of tlv file + * @param[in] bRewind 0 - do not rewind at end of procesing, 1 - do so + * @param[out] bs block signature record + * @param[out] bHasRecHashes 0 if record hashes are present, 1 otherwise + * @param[out] bHasIntermedHashes 0 if intermediate hashes are present, + * 1 otherwise + * + * @returns 0 if ok, something else otherwise + */ +int +rsgt_getBlockParams(FILE *fp, uint8_t bRewind, block_sig_t **bs, + uint8_t *bHasRecHashes, uint8_t *bHasIntermedHashes) +{ + int r; + uint64_t nRecs = 0; + uint8_t bDone = 0; + off_t rewindPos = 0; + void *obj; + tlvrecord_t rec; + + if(bRewind) + rewindPos = ftello(fp); + *bHasRecHashes = 0; + *bHasIntermedHashes = 0; + *bs = NULL; + + while(!bDone) { /* we will err out on EOF */ + if((r = rsgt_tlvrd(fp, &rec, &obj)) != 0) goto done; + switch(rec.tlvtype) { + case 0x0900: + ++nRecs; + *bHasRecHashes = 1; + break; + case 0x0901: + *bHasIntermedHashes = 1; + break; + case 0x0902: + *bs = (block_sig_t*) obj; + bDone = 1; + break; + default:fprintf(fp, "unknown tlv record %4.4x\n", rec.tlvtype); + break; + } + if(!bDone) + rsgt_objfree(rec.tlvtype, obj); + } + + if(*bHasRecHashes && (nRecs != (*bs)->recCount)) { + r = RSGTE_INVLD_RECCNT; + goto done; + } + + if(bRewind) { + if(fseeko(fp, rewindPos, SEEK_SET) != 0) { + r = RSGTE_IO; + goto done; + } + } +done: + return r; +} + + +/** + * Read the file header and compare it to the expected value. + * The file pointer is placed right after the header. + * @param[in] fp file pointer of tlv file + * @param[in] excpect expected header (e.g. "LOGSIG10") + * @returns 0 if ok, something else otherwise + */ +int +rsgt_chkFileHdr(FILE *fp, char *expect) +{ + int r; + char hdr[9]; + + if((r = rsgt_tlvrdHeader(fp, (uchar*)hdr)) != 0) goto done; + if(strcmp(hdr, expect)) + r = RSGTE_INVLHDR; + else + r = 0; +done: + return r; +} + +gtfile +rsgt_vrfyConstruct_gf(void) +{ + gtfile gf; + if((gf = calloc(1, sizeof(struct gtfile_s))) == NULL) + goto done; + gf->x_prev = NULL; + +done: return gf; +} + +void +rsgt_vrfyBlkInit(gtfile gf, block_sig_t *bs, uint8_t bHasRecHashes, uint8_t bHasIntermedHashes) +{ + gf->hashAlg = hashID2Alg(bs->hashID); + gf->bKeepRecordHashes = bHasRecHashes; + gf->bKeepTreeHashes = bHasIntermedHashes; + free(gf->IV); + gf->IV = malloc(getIVLen(bs)); + memcpy(gf->IV, bs->iv, getIVLen(bs)); + free(gf->blkStrtHash); + gf->lenBlkStrtHash = bs->lastHash.len; + gf->blkStrtHash = malloc(gf->lenBlkStrtHash); + memcpy(gf->blkStrtHash, bs->lastHash.data, gf->lenBlkStrtHash); +} + +static int +rsgt_vrfy_chkRecHash(gtfile gf, FILE *sigfp, FILE *nsigfp, + GTDataHash *recHash, gterrctx_t *ectx) +{ + int r = 0; + imprint_t *imp = NULL; + + if((r = rsgt_tlvrdRecHash(sigfp, nsigfp, &imp)) != 0) + reportError(r, ectx); + goto done; + if(imp->hashID != hashIdentifier(gf->hashAlg)) { + reportError(r, ectx); + r = RSGTE_INVLD_REC_HASHID; + goto done; + } + if(memcmp(imp->data, recHash->digest, + hashOutputLengthOctets(imp->hashID))) { + r = RSGTE_INVLD_REC_HASH; + ectx->computedHash = recHash; + ectx->fileHash = imp; + reportError(r, ectx); + ectx->computedHash = NULL, ectx->fileHash = NULL; + goto done; + } + r = 0; +done: + if(imp != NULL) + rsgt_objfree(0x0900, imp); + return r; +} + +static int +rsgt_vrfy_chkTreeHash(gtfile gf, FILE *sigfp, FILE *nsigfp, + GTDataHash *hash, gterrctx_t *ectx) +{ + int r = 0; + imprint_t *imp = NULL; + + if((r = rsgt_tlvrdTreeHash(sigfp, nsigfp, &imp)) != 0) { + reportError(r, ectx); + goto done; + } + if(imp->hashID != hashIdentifier(gf->hashAlg)) { + reportError(r, ectx); + r = RSGTE_INVLD_TREE_HASHID; + goto done; + } + if(memcmp(imp->data, hash->digest, + hashOutputLengthOctets(imp->hashID))) { + r = RSGTE_INVLD_TREE_HASH; + ectx->computedHash = hash; + ectx->fileHash = imp; + reportError(r, ectx); + ectx->computedHash = NULL, ectx->fileHash = NULL; + goto done; + } + r = 0; +done: + if(imp != NULL) + rsgt_objfree(0x0901, imp); + return r; +} + +int +rsgt_vrfy_nextRec(block_sig_t *bs, gtfile gf, FILE *sigfp, FILE *nsigfp, + unsigned char *rec, size_t len, gterrctx_t *ectx) +{ + int r = 0; + GTDataHash *x; /* current hash */ + GTDataHash *m, *recHash = NULL, *t, *t_del; + uint8_t j; + + hash_m(gf, &m); + hash_r(gf, &recHash, rec, len); + if(gf->bKeepRecordHashes) { + r = rsgt_vrfy_chkRecHash(gf, sigfp, nsigfp, recHash, ectx); + if(r != 0) goto done; + } + hash_node(gf, &x, m, recHash, 1); /* hash leaf */ + if(gf->bKeepTreeHashes) { + ectx->treeLevel = 0; + ectx->lefthash = m; + ectx->righthash = recHash; + r = rsgt_vrfy_chkTreeHash(gf, sigfp, nsigfp, x, ectx); + if(r != 0) goto done; + } + rsgtimprintDel(gf->x_prev); + gf->x_prev = rsgtImprintFromGTDataHash(x); + /* add x to the forest as new leaf, update roots list */ + t = x; + for(j = 0 ; j < gf->nRoots ; ++j) { + if(gf->roots_valid[j] == 0) { + gf->roots_hash[j] = t; + gf->roots_valid[j] = 1; + t = NULL; + break; + } else if(t != NULL) { + /* hash interim node */ + ectx->treeLevel = j+1; + ectx->righthash = t; + t_del = t; + hash_node(gf, &t, gf->roots_hash[j], t_del, j+2); + gf->roots_valid[j] = 0; + if(gf->bKeepTreeHashes) { + ectx->lefthash = gf->roots_hash[j]; + r = rsgt_vrfy_chkTreeHash(gf, sigfp, nsigfp, t, ectx); + if(r != 0) goto done; /* mem leak ok, we terminate! */ + } + GTDataHash_free(gf->roots_hash[j]); + GTDataHash_free(t_del); + } + } + if(t != NULL) { + /* new level, append "at the top" */ + gf->roots_hash[gf->nRoots] = t; + gf->roots_valid[gf->nRoots] = 1; + ++gf->nRoots; + assert(gf->nRoots < MAX_ROOTS); + t = NULL; + } + ++gf->nRecords; + + /* cleanup */ + GTDataHash_free(m); +done: + if(recHash != NULL) + GTDataHash_free(recHash); + return r; +} + + +/* TODO: think about merging this with the writer. The + * same applies to the other computation algos. + */ +static int +verifySigblkFinish(gtfile gf, GTDataHash **pRoot) +{ + GTDataHash *root, *rootDel; + int8_t j; + int r; + + if(gf->nRecords == 0) + goto done; + + root = NULL; + for(j = 0 ; j < gf->nRoots ; ++j) { + if(root == NULL) { + root = gf->roots_valid[j] ? gf->roots_hash[j] : NULL; + gf->roots_valid[j] = 0; /* guess this is redundant with init, maybe del */ + } else if(gf->roots_valid[j]) { + rootDel = root; + hash_node(gf, &root, gf->roots_hash[j], root, j+2); + gf->roots_valid[j] = 0; /* guess this is redundant with init, maybe del */ + GTDataHash_free(rootDel); + } + } + + free(gf->blkStrtHash); + gf->blkStrtHash = NULL; + *pRoot = root; + r = 0; +done: + gf->bInBlk = 0; + return r; +} + + +/* helper for rsgt_extendSig: */ +#define COPY_SUBREC_TO_NEWREC \ + memcpy(newrec.data+iWr, subrec.hdr, subrec.lenHdr); \ + iWr += subrec.lenHdr; \ + memcpy(newrec.data+iWr, subrec.data, subrec.tlvlen); \ + iWr += subrec.tlvlen; +static inline int +rsgt_extendSig(GTTimestamp *timestamp, tlvrecord_t *rec, gterrctx_t *ectx) +{ + GTTimestamp *out_timestamp; + uint8_t *der; + size_t lenDer; + int r, rgt; + tlvrecord_t newrec, subrec; + uint16_t iRd, iWr; + + rgt = GTHTTP_extendTimestamp(timestamp, rsgt_extend_puburl, &out_timestamp); + if(rgt != GT_OK) { + ectx->gtstate = rgt; + r = RSGTE_TS_EXTEND; + goto done; + } + r = GTTimestamp_getDEREncoded(out_timestamp, &der, &lenDer); + if(r != GT_OK) { + r = RSGTE_TS_DERENCODE; + ectx->gtstate = rgt; + goto done; + } + /* update block_sig tlv record with new extended timestamp */ + /* we now need to copy all tlv records before the actual der + * encoded part. + */ + iRd = iWr = 0; + // TODO; check tlvtypes at comment places below! + if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + /* HASH_ALGO */ + COPY_SUBREC_TO_NEWREC + if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + /* BLOCK_IV */ + COPY_SUBREC_TO_NEWREC + if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + /* LAST_HASH */ + COPY_SUBREC_TO_NEWREC + if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + /* REC_COUNT */ + COPY_SUBREC_TO_NEWREC + if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + /* actual sig! */ + newrec.data[iWr++] = 0x09 | RSGT_FLAG_TLV16; + newrec.data[iWr++] = 0x06; + newrec.data[iWr++] = (lenDer >> 8) & 0xff; + newrec.data[iWr++] = lenDer & 0xff; + /* now we know how large the new main record is */ + newrec.tlvlen = (uint16_t) iWr+lenDer; + newrec.tlvtype = rec->tlvtype; + newrec.hdr[0] = rec->hdr[0]; + newrec.hdr[1] = rec->hdr[1]; + newrec.hdr[2] = (newrec.tlvlen >> 8) & 0xff; + newrec.hdr[3] = newrec.tlvlen & 0xff; + newrec.lenHdr = 4; + memcpy(newrec.data+iWr, der, lenDer); + /* and finally copy back new record to existing one */ + memcpy(rec, &newrec, sizeof(newrec)-sizeof(newrec.data)+newrec.tlvlen+4); + r = 0; +done: + return r; +} + + +/* verify the root hash. This also means we need to compute the + * Merkle tree root for the current block. + */ +int +verifyBLOCK_SIG(block_sig_t *bs, gtfile gf, FILE *sigfp, FILE *nsigfp, + uint8_t bExtend, gterrctx_t *ectx) +{ + int r; + int gtstate; + block_sig_t *file_bs = NULL; + GTTimestamp *timestamp = NULL; + GTVerificationInfo *vrfyInf; + GTDataHash *root = NULL; + tlvrecord_t rec; + + if((r = verifySigblkFinish(gf, &root)) != 0) + goto done; + if((r = rsgt_tlvrdVrfyBlockSig(sigfp, &file_bs, &rec)) != 0) + goto done; + if(ectx->recNum != bs->recCount) { + r = RSGTE_INVLD_RECCNT; + goto done; + } + + gtstate = GTTimestamp_DERDecode(file_bs->sig.der.data, + file_bs->sig.der.len, ×tamp); + if(gtstate != GT_OK) { + r = RSGTE_TS_DERDECODE; + ectx->gtstate = gtstate; + goto done; + } + + gtstate = GTHTTP_verifyTimestampHash(timestamp, root, NULL, + NULL, NULL, rsgt_read_puburl, 0, &vrfyInf); + if(! (gtstate == GT_OK + && vrfyInf->verification_errors == GT_NO_FAILURES) ) { + r = RSGTE_INVLD_TIMESTAMP; + ectx->gtstate = gtstate; + goto done; + } + + if(rsgt_read_showVerified) + reportVerifySuccess(ectx, vrfyInf); + if(bExtend) + if((r = rsgt_extendSig(timestamp, &rec, ectx)) != 0) goto done; + + if(nsigfp != NULL) + if((r = rsgt_tlvwrite(nsigfp, &rec)) != 0) goto done; + r = 0; +done: + if(file_bs != NULL) + rsgt_objfree(0x0902, file_bs); + if(r != 0) + reportError(r, ectx); + if(timestamp != NULL) + GTTimestamp_free(timestamp); + return r; +} diff --git a/runtime/lmcry_gcry.c b/runtime/lmcry_gcry.c new file mode 100644 index 00000000..0a9b94bc --- /dev/null +++ b/runtime/lmcry_gcry.c @@ -0,0 +1,285 @@ +/* lmcry_gcry.c + * + * An implementation of the cryprov interface for libgcrypt. + * + * Copyright 2013 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" + +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "cryprov.h" +#include "libgcry.h" +#include "lmcry_gcry.h" + +MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + +/* tables for interfacing with the v6 config system */ +static struct cnfparamdescr cnfpdescr[] = { + { "cry.key", eCmdHdlrGetWord, 0 }, + { "cry.keyfile", eCmdHdlrGetWord, 0 }, + { "cry.keyprogram", eCmdHdlrGetWord, 0 }, + { "cry.mode", eCmdHdlrGetWord, 0 }, /* CBC, ECB, etc */ + { "cry.algo", eCmdHdlrGetWord, 0 } +}; +static struct cnfparamblk pblk = + { CNFPARAMBLK_VERSION, + sizeof(cnfpdescr)/sizeof(struct cnfparamdescr), + cnfpdescr + }; + + +#if 0 +static void +errfunc(__attribute__((unused)) void *usrptr, uchar *emsg) +{ + errmsg.LogError(0, RS_RET_CRYPROV_ERR, "Crypto Provider" + "Error: %s - disabling encryption", emsg); +} +#endif + +/* Standard-Constructor + */ +BEGINobjConstruct(lmcry_gcry) + pThis->ctx = gcryCtxNew(); +ENDobjConstruct(lmcry_gcry) + + +/* destructor for the lmcry_gcry object */ +BEGINobjDestruct(lmcry_gcry) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(lmcry_gcry) + rsgcryCtxDel(pThis->ctx); +ENDobjDestruct(lmcry_gcry) + + +/* apply all params from param block to us. This must be called + * after construction, but before the OnFileOpen() entry point. + * Defaults are expected to have been set during construction. + */ +static rsRetVal +SetCnfParam(void *pT, struct nvlst *lst) +{ + lmcry_gcry_t *pThis = (lmcry_gcry_t*) pT; + int i, r; + unsigned keylen; + uchar *key = NULL; + uchar *keyfile = NULL; + uchar *keyprogram = NULL; + uchar *algo = NULL; + uchar *mode = NULL; + int nKeys; /* number of keys (actually methods) specified */ + struct cnfparamvals *pvals; + DEFiRet; + + nKeys = 0; + pvals = nvlstGetParams(lst, &pblk, NULL); + if(Debug) { + dbgprintf("param blk in lmcry_gcry:\n"); + cnfparamsPrint(&pblk, pvals); + } + + for(i = 0 ; i < pblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(pblk.descr[i].name, "cry.key")) { + key = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + ++nKeys; + } else if(!strcmp(pblk.descr[i].name, "cry.keyfile")) { + keyfile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + ++nKeys; + } else if(!strcmp(pblk.descr[i].name, "cry.keyprogram")) { + keyprogram = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + ++nKeys; + } else if(!strcmp(pblk.descr[i].name, "cry.mode")) { + mode = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(pblk.descr[i].name, "cry.algo")) { + algo = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else { + DBGPRINTF("lmcry_gcry: program error, non-handled " + "param '%s'\n", pblk.descr[i].name); + } + } + if(algo != NULL) { + iRet = rsgcrySetAlgo(pThis->ctx, algo); + if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "cry.algo '%s' is not know/supported", algo); + FINALIZE; + } + } + if(mode != NULL) { + iRet = rsgcrySetMode(pThis->ctx, mode); + if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "cry.mode '%s' is not know/supported", mode); + FINALIZE; + } + } + /* note: key must be set AFTER algo/mode is set (as it depends on them) */ + if(nKeys != 1) { + errmsg.LogError(0, RS_RET_INVALID_PARAMS, "excactly one of the following " + "parameters can be specified: cry.key, cry.keyfile, cry.keyprogram\n"); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + if(key != NULL) { + errmsg.LogError(0, RS_RET_ERR, "Note: specifying an actual key directly from the " + "config file is highly insecure - DO NOT USE FOR PRODUCTION"); + keylen = strlen((char*)key); + } + if(keyfile != NULL) { + r = gcryGetKeyFromFile((char*)keyfile, (char**)&key, &keylen); + if(r != 0) { + errmsg.LogError(0, RS_RET_ERR, "error %d reading keyfile %s\n", + r, keyfile); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + } + if(keyprogram != NULL) { + r = gcryGetKeyFromProg((char*)keyprogram, (char**)&key, &keylen); + if(r != 0) { + errmsg.LogError(0, RS_RET_ERR, "error %d obtaining key from program %s\n", + r, keyprogram); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + } + + /* if we reach this point, we have a valid key */ + r = rsgcrySetKey(pThis->ctx, key, keylen); + if(r > 0) { + errmsg.LogError(0, RS_RET_INVALID_PARAMS, "Key length %d expected, but " + "key of length %d given", r, keylen); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + + cnfparamvalsDestruct(pvals, &pblk); + if(key != NULL) { + memset(key, 0, strlen((char*)key)); + free(key); + } + free(keyfile); + free(algo); + free(mode); +finalize_it: + RETiRet; +} + + +static rsRetVal +OnFileOpen(void *pT, uchar *fn, void *pGF) +{ + lmcry_gcry_t *pThis = (lmcry_gcry_t*) pT; + gcryfile *pgf = (gcryfile*) pGF; + DEFiRet; + + CHKiRet(rsgcryInitCrypt(pThis->ctx, pgf, fn)); +finalize_it: + /* TODO: enable this error message (need to cleanup loop first ;)) + errmsg.LogError(0, iRet, "Encryption Provider" + "Error: cannot open .encinfo file - disabling log file"); + */ + RETiRet; +} + +static rsRetVal +Encrypt(void *pF, uchar *rec, size_t *lenRec) +{ + DEFiRet; + iRet = rsgcryEncrypt(pF, rec, lenRec); + + RETiRet; +} + +static rsRetVal +OnFileClose(void *pF, off64_t offsLogfile) +{ + DEFiRet; + gcryfileDestruct(pF, offsLogfile); + + RETiRet; +} + +BEGINobjQueryInterface(lmcry_gcry) +CODESTARTobjQueryInterface(lmcry_gcry) + if(pIf->ifVersion != cryprovCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + pIf->Construct = (rsRetVal(*)(void*)) lmcry_gcryConstruct; + pIf->SetCnfParam = SetCnfParam; + pIf->Destruct = (rsRetVal(*)(void*)) lmcry_gcryDestruct; + pIf->OnFileOpen = OnFileOpen; + pIf->Encrypt = Encrypt; + pIf->OnFileClose = OnFileClose; +finalize_it: +ENDobjQueryInterface(lmcry_gcry) + + +BEGINObjClassExit(lmcry_gcry, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(lmcry_gcry) + /* release objects we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + + rsgcryExit(); +ENDObjClassExit(lmcry_gcry) + + +BEGINObjClassInit(lmcry_gcry, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + if(rsgcryInit() != 0) { + errmsg.LogError(0, RS_RET_CRYPROV_ERR, "error initializing " + "crypto provider - cannot encrypt"); + ABORT_FINALIZE(RS_RET_CRYPROV_ERR); + } +ENDObjClassInit(lmcry_gcry) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit + lmcry_gcryClassExit(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + /* Initialize all classes that are in our module - this includes ourselfs */ + CHKiRet(lmcry_gcryClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ +ENDmodInit diff --git a/runtime/lmcry_gcry.h b/runtime/lmcry_gcry.h new file mode 100644 index 00000000..c0205ab9 --- /dev/null +++ b/runtime/lmcry_gcry.h @@ -0,0 +1,39 @@ +/* An implementation of the cryprov interface for libgcrypt. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_LMCRY_GCRY_H +#define INCLUDED_LMCRY_GCRY_H +#include "cryprov.h" + +/* interface is defined in cryprov.h, we just implement it! */ +#define lmcry_gcryCURR_IF_VERSION cryprovCURR_IF_VERSION +typedef cryprov_if_t lmcry_gcry_if_t; + +/* the lmcry_gcry object */ +struct lmcry_gcry_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + gcryctx ctx; +}; +typedef struct lmcry_gcry_s lmcry_gcry_t; + +/* prototypes */ +PROTOTYPEObj(lmcry_gcry); + +#endif /* #ifndef INCLUDED_LMCRY_GCRY_H */ diff --git a/runtime/lmsig_gt.c b/runtime/lmsig_gt.c new file mode 100644 index 00000000..116a48d5 --- /dev/null +++ b/runtime/lmsig_gt.c @@ -0,0 +1,233 @@ +/* lmsig_gt.c + * + * An implementation of the sigprov interface for GuardTime. + * + * Copyright 2013 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" + +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "sigprov.h" +#include "lmsig_gt.h" + +MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + +/* tables for interfacing with the v6 config system */ +static struct cnfparamdescr cnfpdescr[] = { + { "sig.hashfunction", eCmdHdlrGetWord, 0 }, + { "sig.timestampservice", eCmdHdlrGetWord, 0 }, + { "sig.block.sizelimit", eCmdHdlrSize, 0 }, + { "sig.keeprecordhashes", eCmdHdlrBinary, 0 }, + { "sig.keeptreehashes", eCmdHdlrBinary, 0 } +}; +static struct cnfparamblk pblk = + { CNFPARAMBLK_VERSION, + sizeof(cnfpdescr)/sizeof(struct cnfparamdescr), + cnfpdescr + }; + + +static void +errfunc(__attribute__((unused)) void *usrptr, uchar *emsg) +{ + errmsg.LogError(0, RS_RET_SIGPROV_ERR, "Signature Provider" + "Error: %s - disabling signatures", emsg); +} + +/* Standard-Constructor + */ +BEGINobjConstruct(lmsig_gt) + pThis->ctx = rsgtCtxNew(); + rsgtsetErrFunc(pThis->ctx, errfunc, NULL); +ENDobjConstruct(lmsig_gt) + + +/* destructor for the lmsig_gt object */ +BEGINobjDestruct(lmsig_gt) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(lmsig_gt) + rsgtCtxDel(pThis->ctx); +ENDobjDestruct(lmsig_gt) + + +/* apply all params from param block to us. This must be called + * after construction, but before the OnFileOpen() entry point. + * Defaults are expected to have been set during construction. + */ +rsRetVal +SetCnfParam(void *pT, struct nvlst *lst) +{ + lmsig_gt_t *pThis = (lmsig_gt_t*) pT; + int i; + uchar *cstr; + struct cnfparamvals *pvals; + DEFiRet; + pvals = nvlstGetParams(lst, &pblk, NULL); + if(pvals == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + if(Debug) { + dbgprintf("sig param blk in lmsig_gt:\n"); + cnfparamsPrint(&pblk, pvals); + } + + for(i = 0 ; i < pblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(pblk.descr[i].name, "sig.hashfunction")) { + cstr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + if(rsgtSetHashFunction(pThis->ctx, (char*)cstr) != 0) { + errmsg.LogError(0, RS_RET_ERR, "Hash function " + "'%s' unknown - using default", cstr); + } + free(cstr); + } else if(!strcmp(pblk.descr[i].name, "sig.timestampservice")) { + cstr = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + rsgtSetTimestamper(pThis->ctx, (char*) cstr); + free(cstr); + } else if(!strcmp(pblk.descr[i].name, "sig.block.sizelimit")) { + rsgtSetBlockSizeLimit(pThis->ctx, pvals[i].val.d.n); + } else if(!strcmp(pblk.descr[i].name, "sig.keeprecordhashes")) { + rsgtSetKeepRecordHashes(pThis->ctx, pvals[i].val.d.n); + } else if(!strcmp(pblk.descr[i].name, "sig.keeptreehashes")) { + rsgtSetKeepTreeHashes(pThis->ctx, pvals[i].val.d.n); + } else { + DBGPRINTF("lmsig_gt: program error, non-handled " + "param '%s'\n", pblk.descr[i].name); + } + } +finalize_it: + if(pvals != NULL) + cnfparamvalsDestruct(pvals, &pblk); + RETiRet; +} + + +static rsRetVal +OnFileOpen(void *pT, uchar *fn, void *pGF) +{ + lmsig_gt_t *pThis = (lmsig_gt_t*) pT; + gtfile *pgf = (gtfile*) pGF; + DEFiRet; + DBGPRINTF("lmsig_gt: onFileOpen: %s\n", fn); + /* note: if *pgf is set to NULL, this auto-disables GT functions */ + *pgf = rsgtCtxOpenFile(pThis->ctx, fn); + sigblkInit(*pgf); + RETiRet; +} + +/* Note: we assume that the record is terminated by a \n. + * As of the GuardTime paper, \n is not part of the signed + * message, so we subtract one from the record size. This + * may cause issues with non-standard formats, but let's + * see how things evolve (the verifier will not work in + * any case when the records are not \n delimited...). + * rgerhards, 2013-03-17 + */ +static rsRetVal +OnRecordWrite(void *pF, uchar *rec, rs_size_t lenRec) +{ + DEFiRet; + DBGPRINTF("lmsig_gt: onRecordWrite (%d): %s\n", lenRec-1, rec); + sigblkAddRecord(pF, rec, lenRec-1); + + RETiRet; +} + +static rsRetVal +OnFileClose(void *pF) +{ + DEFiRet; + DBGPRINTF("lmsig_gt: onFileClose\n"); + rsgtfileDestruct(pF); + + RETiRet; +} + +BEGINobjQueryInterface(lmsig_gt) +CODESTARTobjQueryInterface(lmsig_gt) + if(pIf->ifVersion != sigprovCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + pIf->Construct = (rsRetVal(*)(void*)) lmsig_gtConstruct; + pIf->SetCnfParam = SetCnfParam; + pIf->Destruct = (rsRetVal(*)(void*)) lmsig_gtDestruct; + pIf->OnFileOpen = OnFileOpen; + pIf->OnRecordWrite = OnRecordWrite; + pIf->OnFileClose = OnFileClose; +finalize_it: +ENDobjQueryInterface(lmsig_gt) + + +BEGINObjClassExit(lmsig_gt, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(lmsig_gt) + /* release objects we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + + rsgtExit(); +ENDObjClassExit(lmsig_gt) + + +BEGINObjClassInit(lmsig_gt, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + if(rsgtInit("rsyslogd " VERSION) != 0) { + errmsg.LogError(0, RS_RET_SIGPROV_ERR, "error initializing " + "signature provider - cannot sign"); + ABORT_FINALIZE(RS_RET_SIGPROV_ERR); + } +ENDObjClassInit(lmsig_gt) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit + lmsig_gtClassExit(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + /* Initialize all classes that are in our module - this includes ourselfs */ + CHKiRet(lmsig_gtClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ +ENDmodInit diff --git a/runtime/lmsig_gt.h b/runtime/lmsig_gt.h new file mode 100644 index 00000000..665e6a8e --- /dev/null +++ b/runtime/lmsig_gt.h @@ -0,0 +1,40 @@ +/* An implementation of the sigprov interface for GuardTime. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_LMSIG_GT_H +#define INCLUDED_LMSIG_GT_H +#include "sigprov.h" +#include "librsgt.h" + +/* interface is defined in sigprov.h, we just implement it! */ +#define lmsig_gtCURR_IF_VERSION sigprovCURR_IF_VERSION +typedef sigprov_if_t lmsig_gt_if_t; + +/* the lmsig_gt object */ +struct lmsig_gt_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + gtctx ctx; /* librsgt context - contains all we need */ +}; +typedef struct lmsig_gt_s lmsig_gt_t; + +/* prototypes */ +PROTOTYPEObj(lmsig_gt); + +#endif /* #ifndef INCLUDED_LMSIG_GT_H */ diff --git a/runtime/module-template.h b/runtime/module-template.h index 9dd759a5..8a958f90 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -113,7 +113,7 @@ static rsRetVal modGetID(void **pID) \ /* macro to provide the v6 config system module name */ #define MODULE_CNFNAME(name) \ -static __attribute__((unused)) rsRetVal modGetCnfName(uchar **cnfName) \ +static rsRetVal modGetCnfName(uchar **cnfName) \ { \ *cnfName = (uchar*) name; \ return RS_RET_OK;\ @@ -246,7 +246,8 @@ static rsRetVal dbgPrintInstInfo(void *pModData)\ instanceData *pData = NULL; #define CODESTARTdbgPrintInstInfo \ - pData = (instanceData*) pModData; + pData = (instanceData*) pModData; \ + (void)pData; /* prevent compiler warning if unused! */ #define ENDdbgPrintInstInfo \ RETiRet;\ @@ -937,6 +938,28 @@ static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\ } +/* SetShutdownImmdtPtr() + * This function is optional. If defined by an output plugin, it is called + * each time the action is invoked to set the "ShutdownImmediate" pointer, + * which is used during termination to indicate the action should shutdown + * as quickly as possible. + */ +#define CODEqueryEtryPt_SetShutdownImmdtPtr \ + else if(!strcmp((char*) name, "SetShutdownImmdtPtr")) {\ + *pEtryPoint = SetShutdownImmdtPtr;\ + } +#define BEGINSetShutdownImmdtPtr \ +static rsRetVal SetShutdownImmdtPtr(instanceData __attribute__((unused)) *pData, int *pPtr)\ +{\ + DEFiRet; + +#define CODESTARTSetShutdownImmdtPtr + +#define ENDSetShutdownImmdtPtr \ + RETiRet;\ +} + + /* parse() - main entry point of parser modules */ #define BEGINparse \ diff --git a/runtime/modules.c b/runtime/modules.c index 5706685f..56606306 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -313,7 +313,8 @@ finalize_it: /* get the name of a module */ -static uchar *modGetName(modInfo_t *pThis) +uchar * +modGetName(modInfo_t *pThis) { return((pThis->pszName == NULL) ? (uchar*) "" : pThis->pszName); } @@ -656,6 +657,10 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) ABORT_FINALIZE(localRet); + localRet = (*pNew->modQueryEtryPt)((uchar*)"SetShutdownImmdtPtr", &pNew->mod.om.SetShutdownImmdtPtr); + if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + ABORT_FINALIZE(localRet); + localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction); if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) pNew->mod.om.beginTransaction = dummyBeginTransaction; @@ -1040,7 +1045,6 @@ Load(uchar *pModName, sbool bConfLoad, struct nvlst *lst) if(bConfLoad) { localRet = readyModForCnf(pModInfo, &pNew, &pLast); if(pModInfo->setModCnf != NULL && localRet == RS_RET_OK) { - addModToCnfList(pNew, pLast); if(!strncmp((char*)pModName, "builtin:", sizeof("builtin:")-1)) { if(pModInfo->bSetModCnfCalled) { errmsg.LogError(0, RS_RET_DUP_PARAM, @@ -1056,6 +1060,11 @@ Load(uchar *pModName, sbool bConfLoad, struct nvlst *lst) pModInfo->setModCnf(lst); pModInfo->bSetModCnfCalled = 1; } + } else { + /* regular modules need to be added to conf list (for + * builtins, this happend during initial load). + */ + addModToCnfList(pNew, pLast); } } } diff --git a/runtime/modules.h b/runtime/modules.h index 02e4a699..64644be2 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -142,6 +142,7 @@ struct modInfo_s { rsRetVal (*endTransaction)(void*); rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); rsRetVal (*newActInst)(uchar *modName, struct nvlst *lst, void **, omodStringRequest_t **); + rsRetVal (*SetShutdownImmdtPtr)(void *pData, void *pPtr); } om; struct { /* data for library modules */ char dummy; @@ -190,8 +191,11 @@ ENDinterface(module) PROTOTYPEObj(module); /* in v6, we go back to in-core static link for core objects, at least those * that are not called from plugins. + * ... and we need to know that none of the module functions are called from plugins! + * rgerhards, 2012-09-24 */ rsRetVal modulesProcessCnf(struct cnfobj *o); +uchar *modGetName(modInfo_t *pThis); rsRetVal addModToCnfList(cfgmodules_etry_t *pNew, cfgmodules_etry_t *pLast); rsRetVal readyModForCnf(modInfo_t *pThis, cfgmodules_etry_t **ppNew, cfgmodules_etry_t **ppLast); #endif /* #ifndef MODULES_H_INCLUDED */ diff --git a/runtime/msg.c b/runtime/msg.c index 45ebf5f9..67d957d1 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -7,7 +7,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -36,14 +36,20 @@ #include <assert.h> #include <ctype.h> #include <sys/socket.h> +#if HAVE_SYSINFO_UPTIME #include <sys/sysinfo.h> +#endif #include <netdb.h> #include <libestr.h> -#include <libee/libee.h> +#include <json/json.h> +/* For struct json_object_iter, should not be necessary in future versions */ +#include <json/json_object_private.h> #if HAVE_MALLOC_H # include <malloc.h> #endif -#include <uuid/uuid.h> +#ifdef USE_LIBUUID + #include <uuid/uuid.h> +#endif #include "rsyslog.h" #include "srUtils.h" #include "stringbuf.h" @@ -57,6 +63,7 @@ #include "ruleset.h" #include "prop.h" #include "net.h" +#include "var.h" #include "rsconf.h" /* static data */ @@ -66,6 +73,19 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) DEFobjCurrIf(prop) DEFobjCurrIf(net) +DEFobjCurrIf(var) + +static char *two_digits[100] = { + "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", + "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", + "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", + "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", + "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", + "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", + "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", + "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", + "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", + "90", "91", "92", "93", "94", "95", "96", "97", "98", "99"}; static struct { uchar *pszName; @@ -273,9 +293,15 @@ static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", "news", "uucp", "cron", "authpriv", "ftp", "ntp", "audit", "alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7" }; +/* length of the facility names string (for optimizatiions) */ +static short len_syslog_fac_names[24] = { 4, 4, 4, 6, 4, 6, 3, + 4, 4, 4, 8, 3, 3, 5, + 5, 5, 6, 6, 6, 6, + 6, 6, 6, 6 }; /* table of severity names (in numerical order)*/ static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" }; +static short len_syslog_severity_names[8] = { 5, 5, 4, 3, 7, 6, 4, 5 }; /* numerical values as string - this is the most efficient approach to convert severity * and facility values to a numerical string... -- rgerhars, 2009-06-17 @@ -291,6 +317,9 @@ static pthread_mutex_t mutTrimCtr; /* mutex to handle malloc trim */ /* some forward declarations */ static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); +static rsRetVal jsonPathFindParent(msg_t *pM, uchar *name, uchar *leaf, struct json_object **parent, int bCreate); +static uchar * jsonPathGetLeaf(uchar *name, int lenName); +static struct json_object *jsonDeepCopy(struct json_object *src); /* the locking and unlocking implementations: */ @@ -308,6 +337,47 @@ MsgUnlock(msg_t *pThis) } +/* set RcvFromIP name in msg object WITHOUT calling AddRef. + * rgerhards, 2013-01-22 + */ +static inline void +MsgSetRcvFromIPWithoutAddRef(msg_t *pThis, prop_t *new) +{ + if(pThis->pRcvFromIP != NULL) + prop.Destruct(&pThis->pRcvFromIP); + pThis->pRcvFromIP = new; +} + + +/* set RcvFrom name in msg object WITHOUT calling AddRef. + * rgerhards, 2013-01-22 + */ +void MsgSetRcvFromWithoutAddRef(msg_t *pThis, prop_t *new) +{ + assert(pThis != NULL); + + if(pThis->msgFlags & NEEDS_DNSRESOL) { + if(pThis->rcvFrom.pfrominet != NULL) + free(pThis->rcvFrom.pfrominet); + pThis->msgFlags &= ~NEEDS_DNSRESOL; + } else { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } + pThis->rcvFrom.pRcvFrom = new; +} + + +/* rgerhards 2012-04-18: set associated ruleset (by ruleset name) + * If ruleset cannot be found, no update is done. + */ +static void +MsgSetRulesetByName(msg_t *pMsg, cstr_t *rulesetName) +{ + rulesetGetRuleset(runConf, &(pMsg->pRuleset), rsCStrGetSzStrNoNULL(rulesetName)); +} + + static inline int getProtocolVersion(msg_t *pM) { return(pM->iProtocolVersion); @@ -321,19 +391,18 @@ static inline rsRetVal resolveDNS(msg_t *pMsg) { rsRetVal localRet; prop_t *propFromHost = NULL; - prop_t *propFromHostIP = NULL; - uchar fromHost[NI_MAXHOST]; - uchar fromHostIP[NI_MAXHOST]; - uchar fromHostFQDN[NI_MAXHOST]; + prop_t *ip; + prop_t *localName; DEFiRet; MsgLock(pMsg); CHKiRet(objUse(net, CORE_COMPONENT)); if(pMsg->msgFlags & NEEDS_DNSRESOL) { - localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); + localRet = net.cvthname(pMsg->rcvFrom.pfrominet, &localName, NULL, &ip); if(localRet == RS_RET_OK) { - MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); - CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + /* we pass down the props, so no need for AddRef */ + MsgSetRcvFromWithoutAddRef(pMsg, localName); + MsgSetRcvFromIPWithoutAddRef(pMsg, ip); } } finalize_it: @@ -345,8 +414,6 @@ finalize_it: MsgUnlock(pMsg); if(propFromHost != NULL) prop.Destruct(&propFromHost); - if(propFromHostIP != NULL) - prop.Destruct(&propFromHostIP); RETiRet; } @@ -442,8 +509,10 @@ propNameStrToID(uchar *pName, propid_t *pPropID) *pPropID = PROP_MSGID; } else if(!strcmp((char*) pName, "parsesuccess")) { *pPropID = PROP_PARSESUCCESS; +#ifdef USE_LIBUUID } else if(!strcmp((char*) pName, "uuid")) { *pPropID = PROP_UUID; +#endif /* here start system properties (those, that do not relate to the message itself */ } else if(!strcmp((char*) pName, "$now")) { *pPropID = PROP_SYS_NOW; @@ -611,6 +680,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; + pM->iLenPROGNAME = -1; pM->offAfterPRI = 0; pM->offMSG = -1; pM->iProtocolVersion = 0; @@ -629,7 +699,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pszTIMESTAMP3339 = NULL; pM->pszTIMESTAMP_MySQL = NULL; pM->pszTIMESTAMP_PgSQL = NULL; - pM->pCSProgName = NULL; pM->pCSStrucData = NULL; pM->pCSAPPNAME = NULL; pM->pCSPROCID = NULL; @@ -638,7 +707,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pRcvFromIP = NULL; pM->rcvFrom.pRcvFrom = NULL; pM->pRuleset = NULL; - pM->event = NULL; + pM->json = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); pM->TAG.pszTAG = NULL; @@ -708,6 +777,19 @@ finalize_it: } +/* Special msg constructor, to be used when an object is deserialized. + * we do only the base init as we know the properties will be set in + * any case by the deserializer. We still do the "inexpensive" inits + * just to be on the safe side. The whole process needs to be + * refactored together with the msg serialization subsystem. + */ +rsRetVal +msgConstructForDeserializer(msg_t **ppThis) +{ + return msgBaseConstruct(ppThis); +} + + /* some free handlers for (slightly) complicated cases... All of them may be called * with an empty element. */ @@ -759,8 +841,8 @@ CODESTARTobjDestruct(msg) free(pThis->pszRcvdAt_PgSQL); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); - if(pThis->pCSProgName != NULL) - rsCStrDestruct(&pThis->pCSProgName); + if(pThis->iLenPROGNAME >= CONF_PROGNAME_BUFSIZE) + free(pThis->PROGNAME.ptr); if(pThis->pCSStrucData != NULL) rsCStrDestruct(&pThis->pCSStrucData); if(pThis->pCSAPPNAME != NULL) @@ -769,8 +851,8 @@ CODESTARTobjDestruct(msg) rsCStrDestruct(&pThis->pCSPROCID); if(pThis->pCSMSGID != NULL) rsCStrDestruct(&pThis->pCSMSGID); - if(pThis->event != NULL) - ee_deleteEvent(pThis->event); + if(pThis->json != NULL) + json_object_put(pThis->json); if(pThis->pszUUID != NULL) free(pThis->pszUUID); # ifndef HAVE_ATOMIC_BUILTINS @@ -913,12 +995,14 @@ msg_t* MsgDup(msg_t* pOld) } } - tmpCOPYCSTR(ProgName); tmpCOPYCSTR(StrucData); tmpCOPYCSTR(APPNAME); tmpCOPYCSTR(PROCID); tmpCOPYCSTR(MSGID); + if(pOld->json != NULL) + pNew->json = jsonDeepCopy(pOld->json); + /* we do not copy all other cache properties, as we do not even know * if they are needed once again. So we let them re-create if needed. */ @@ -972,6 +1056,10 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFrom"), PROPTYPE_PSZ, (void*) psz)); psz = getRcvFromIP(pThis); CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFromIP"), PROPTYPE_PSZ, (void*) psz)); + if(pThis->json != NULL) { + psz = (uchar*) json_object_get_string(pThis->json); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("json"), PROPTYPE_PSZ, (void*) psz)); + } objSerializePTR(pStrm, pCSStrucData, CSTR); objSerializePTR(pStrm, pCSAPPNAME, CSTR); @@ -998,6 +1086,168 @@ finalize_it: } +/* This is a helper for MsgDeserialize that re-inits the var object. This + * whole construct should be replaced, var is really ready to be retired. + * But as an interim help during refactoring let's introduce this function + * here (and thus NOT as method of var object!). -- rgerhads, 2012-11-06 + */ +static inline void +reinitVar(var_t *pVar) +{ + rsCStrDestruct(&pVar->pcsName); /* no longer needed */ + if(pVar->varType == VARTYPE_STR) { + if(pVar->val.pStr != NULL) + rsCStrDestruct(&pVar->val.pStr); + } +} +/* deserialize the message again + * we deserialize the properties in the same order that we serialized them. Except + * for some checks to cover downlevel version, we do not need to do all these + * CPU intense name checkings. + */ +#define isProp(name) !rsCStrSzStrCmp(pVar->pcsName, (uchar*) name, sizeof(name) - 1) +rsRetVal +MsgDeserialize(msg_t *pMsg, strm_t *pStrm) +{ + prop_t *myProp; + prop_t *propRcvFrom = NULL; + prop_t *propRcvFromIP = NULL; + struct json_tokener *tokener; + var_t *pVar = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pStrm, strm); + + CHKiRet(var.Construct(&pVar)); + CHKiRet(var.ConstructFinalize(pVar)); + + CHKiRet(objDeserializeProperty(pVar, pStrm)); + if(isProp("iProtocolVersion")) { + setProtocolVersion(pMsg, pVar->val.num); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("iSeverity")) { + pMsg->iSeverity = pVar->val.num; + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("iFacility")) { + pMsg->iFacility = pVar->val.num; + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("msgFlags")) { + pMsg->msgFlags = pVar->val.num; + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("ttGenTime")) { + pMsg->ttGenTime = pVar->val.num; + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("tRcvdAt")) { + memcpy(&pMsg->tRcvdAt, &pVar->val.vSyslogTime, sizeof(struct syslogTime)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("tTIMESTAMP")) { + memcpy(&pMsg->tTIMESTAMP, &pVar->val.vSyslogTime, sizeof(struct syslogTime)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszTAG")) { + MsgSetTAG(pMsg, rsCStrGetSzStrNoNULL(pVar->val.pStr), cstrLen(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszRawMsg")) { + MsgSetRawMsg(pMsg, (char*) rsCStrGetSzStrNoNULL(pVar->val.pStr), cstrLen(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszHOSTNAME")) { + MsgSetHOSTNAME(pMsg, rsCStrGetSzStrNoNULL(pVar->val.pStr), rsCStrLen(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszInputName")) { + /* we need to create a property */ + CHKiRet(prop.Construct(&myProp)); + CHKiRet(prop.SetString(myProp, rsCStrGetSzStrNoNULL(pVar->val.pStr), rsCStrLen(pVar->val.pStr))); + CHKiRet(prop.ConstructFinalize(myProp)); + MsgSetInputName(pMsg, myProp); + prop.Destruct(&myProp); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszRcvFrom")) { + MsgSetRcvFromStr(pMsg, rsCStrGetSzStrNoNULL(pVar->val.pStr), rsCStrLen(pVar->val.pStr), &propRcvFrom); + prop.Destruct(&propRcvFrom); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszRcvFromIP")) { + MsgSetRcvFromIPStr(pMsg, rsCStrGetSzStrNoNULL(pVar->val.pStr), rsCStrLen(pVar->val.pStr), &propRcvFromIP); + prop.Destruct(&propRcvFromIP); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("json")) { + tokener = json_tokener_new(); + pMsg->json = json_tokener_parse_ex(tokener, (char*)rsCStrGetSzStrNoNULL(pVar->val.pStr), + cstrLen(pVar->val.pStr)); + json_tokener_free(tokener); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pCSStrucData")) { + MsgSetStructuredData(pMsg, (char*) rsCStrGetSzStrNoNULL(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pCSAPPNAME")) { + MsgSetAPPNAME(pMsg, (char*) rsCStrGetSzStrNoNULL(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pCSPROCID")) { + MsgSetPROCID(pMsg, (char*) rsCStrGetSzStrNoNULL(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pCSMSGID")) { + MsgSetMSGID(pMsg, (char*) rsCStrGetSzStrNoNULL(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszUUID")) { + pMsg->pszUUID = ustrdup(rsCStrGetSzStrNoNULL(pVar->val.pStr)); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + if(isProp("pszRuleset")) { + MsgSetRulesetByName(pMsg, pVar->val.pStr); + reinitVar(pVar); + CHKiRet(objDeserializeProperty(pVar, pStrm)); + } + /* "offMSG" must always be our last field, so we use this as an + * indicator if the sequence is correct. This is a bit questionable, + * but on the other hand it works decently AND we will probably replace + * the whole persisted format soon in any case. -- rgerhards, 2012-11-06 + */ + if(!isProp("offMSG")) + ABORT_FINALIZE(RS_RET_DS_PROP_SEQ_ERR); + MsgSetMSGoffs(pMsg, pVar->val.num); +finalize_it: + if(pVar != NULL) + var.Destruct(&pVar); + RETiRet; +} +#undef isProp + + /* Increment reference count - see description of the "msg" * structure for details. As a convenience to developers, * this method returns the msg pointer that is passed to it. @@ -1092,32 +1342,33 @@ finalize_it: * The above definition has been taken from the FreeBSD syslogd sources. * * The program name is not parsed by default, because it is infrequently-used. - * If it is needed, this function should be called first. It checks if it is - * already set and extracts it, if not. - * * IMPORTANT: A locked message object must be provided, else a crash will occur. * rgerhards, 2005-10-19 */ -static rsRetVal aquireProgramName(msg_t *pM) +static inline rsRetVal +aquireProgramName(msg_t *pM) { - register int i; - uchar *pszTag; + int i; + uchar *pszTag, *pszProgName; DEFiRet; assert(pM != NULL); - if(pM->pCSProgName == NULL) { - /* ok, we do not yet have it. So let's parse the TAG to obtain it. */ - pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); - CHKiRet(cstrConstruct(&pM->pCSProgName)); - for( i = 0 - ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) - && (pszTag[i] != '\0') && (pszTag[i] != ':') - && (pszTag[i] != '[') && (pszTag[i] != '/') - ; ++i) { - CHKiRet(cstrAppendChar(pM->pCSProgName, pszTag[i])); - } - CHKiRet(cstrFinalize(pM->pCSProgName)); + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); + for( i = 0 + ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) + && (pszTag[i] != '\0') && (pszTag[i] != ':') + && (pszTag[i] != '[') && (pszTag[i] != '/') + ; ++i) + ; /* just search end of PROGNAME */ + if(i < CONF_PROGNAME_BUFSIZE) { + pszProgName = pM->PROGNAME.szBuf; + } else { + CHKmalloc(pM->PROGNAME.ptr = malloc(i+1)); + pszProgName = pM->PROGNAME.ptr; } + memcpy((char*)pszProgName, (char*)pszTag, i); + pszProgName[i] = '\0'; + pM->iLenPROGNAME = i; finalize_it: RETiRet; } @@ -1142,6 +1393,7 @@ char *getProtocolVersionString(msg_t *pM) return(pM->iProtocolVersion ? "1" : "0"); } +#ifdef USE_LIBUUID /* note: libuuid seems not to be thread-safe, so we need * to get some safeguards in place. */ @@ -1196,6 +1448,7 @@ void getUUID(msg_t *pM, uchar **pBuf, int *piLen) } dbgprintf("[getUUID] END\n"); } +#endif void getRawMsg(msg_t *pM, uchar **pBuf, int *piLen) @@ -1215,6 +1468,14 @@ getRawMsg(msg_t *pM, uchar **pBuf, int *piLen) } +/* note: setMSGLen() is only for friends who really know what they + * do. Setting an invalid length can be desasterous! + */ +void setMSGLen(msg_t *pM, int lenMsg) +{ + pM->iLenMSG = lenMsg; +} + int getMSGLen(msg_t *pM) { return((pM == NULL) ? 0 : pM->iLenMSG); @@ -1664,16 +1925,6 @@ void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) } -/* rgerhards 2012-04-18: set associated ruleset (by ruleset name) - * If ruleset cannot be found, no update is done. - */ -static void -MsgSetRulesetByName(msg_t *pMsg, cstr_t *rulesetName) -{ - rulesetGetRuleset(runConf, &(pMsg->pRuleset), rsCStrGetSzStrNoNULL(rulesetName)); -} - - /* set TAG in msg object * (rewritten 2009-06-18 rgerhards) */ @@ -1862,53 +2113,24 @@ static inline char *getStructuredData(msg_t *pM) return (char*) pszRet; } -/* check if we have a ProgramName, and, if not, try to aquire/emulate it. - * rgerhards, 2009-06-26 - */ -static inline void prepareProgramName(msg_t *pM, sbool bLockMutex) -{ - if(pM->pCSProgName == NULL) { - if(bLockMutex == LOCK_MUTEX) - MsgLock(pM); - - /* re-query as things might have changed during locking */ - if(pM->pCSProgName == NULL) - aquireProgramName(pM); - - if(bLockMutex == LOCK_MUTEX) - MsgUnlock(pM); - } -} - - -/* get the length of the "programname" sz string - * rgerhards, 2005-10-19 - */ -int getProgramNameLen(msg_t *pM, sbool bLockMutex) -{ - assert(pM != NULL); - prepareProgramName(pM, bLockMutex); - return (pM->pCSProgName == NULL) ? 0 : rsCStrLen(pM->pCSProgName); -} - - /* get the "programname" as sz string * rgerhards, 2005-10-19 */ uchar *getProgramName(msg_t *pM, sbool bLockMutex) { - uchar *pszRet; - - if(bLockMutex == LOCK_MUTEX) - MsgLock(pM); - prepareProgramName(pM, MUTEX_ALREADY_LOCKED); - if(pM->pCSProgName == NULL) - pszRet = UCHAR_CONSTANT(""); - else - pszRet = rsCStrGetSzStrNoNULL(pM->pCSProgName); - if(bLockMutex == LOCK_MUTEX) - MsgUnlock(pM); - return pszRet; + if(pM->iLenPROGNAME == -1) { + if(bLockMutex == LOCK_MUTEX) { + MsgLock(pM); + /* need to re-check, things may have change in between! */ + if(pM->iLenPROGNAME == -1) + aquireProgramName(pM); + MsgUnlock(pM); + } else { + aquireProgramName(pM); + } + } + return (pM->iLenPROGNAME < CONF_PROGNAME_BUFSIZE) ? pM->PROGNAME.szBuf + : pM->PROGNAME.ptr; } @@ -2023,18 +2245,8 @@ finalize_it: */ void MsgSetRcvFrom(msg_t *pThis, prop_t *new) { - assert(pThis != NULL); - prop.AddRef(new); - if(pThis->msgFlags & NEEDS_DNSRESOL) { - if(pThis->rcvFrom.pfrominet != NULL) - free(pThis->rcvFrom.pfrominet); - pThis->msgFlags &= ~NEEDS_DNSRESOL; - } else { - if(pThis->rcvFrom.pRcvFrom != NULL) - prop.Destruct(&pThis->rcvFrom.pRcvFrom); - } - pThis->rcvFrom.pRcvFrom = new; + MsgSetRcvFromWithoutAddRef(pThis, new); } @@ -2067,9 +2279,7 @@ rsRetVal MsgSetRcvFromIP(msg_t *pThis, prop_t *new) BEGINfunc prop.AddRef(new); - if(pThis->pRcvFromIP != NULL) - prop.Destruct(&pThis->pRcvFromIP); - pThis->pRcvFromIP = new; + MsgSetRcvFromIPWithoutAddRef(pThis, new); ENDfunc return RS_RET_OK; } @@ -2223,21 +2433,20 @@ void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg) /* Decode a priority into textual information like auth.emerg. - * The variable pRes must point to a user-supplied buffer and - * pResLen must contain its size. The pointer to the buffer + * The variable pRes must point to a user-supplied buffer. + * The pointer to the buffer * is also returned, what makes this functiona suitable for * use in printf-like functions. * Note: a buffer size of 20 characters is always sufficient. - * Interface to this function changed 2007-06-15 by RGerhards */ -char *textpri(char *pRes, size_t pResLen, int pri) +char *textpri(char *pRes, int pri) { assert(pRes != NULL); - assert(pResLen > 0); - - snprintf(pRes, pResLen, "%s.%s", syslog_fac_names[LOG_FAC(pri)], - syslog_severity_names[LOG_PRI(pri)]); - + memcpy(pRes, syslog_fac_names[LOG_FAC(pri)], len_syslog_fac_names[LOG_FAC(pri)]); + pRes[len_syslog_fac_names[LOG_FAC(pri)]] = '.'; + memcpy(pRes+len_syslog_fac_names[LOG_FAC(pri)]+1, + syslog_severity_names[LOG_PRI(pri)], + len_syslog_severity_names[LOG_PRI(pri)]+1 /* for \0! */); return pRes; } @@ -2251,40 +2460,48 @@ char *textpri(char *pRes, size_t pResLen, int pri) */ typedef enum ENOWType { NOW_NOW, NOW_YEAR, NOW_MONTH, NOW_DAY, NOW_HOUR, NOW_HHOUR, NOW_QHOUR, NOW_MINUTE } eNOWType; #define tmpBUFSIZE 16 /* size of formatting buffer */ -static uchar *getNOW(eNOWType eNow) +static uchar *getNOW(eNOWType eNow, struct syslogTime *t) { uchar *pBuf; - struct syslogTime t; if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { return NULL; } - datetime.getCurrTime(&t, NULL); + if(t->year == 0) { /* not yet set! */ + datetime.getCurrTime(t, NULL); + } + switch(eNow) { case NOW_NOW: - snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day); + memcpy(pBuf, two_digits[t->year/100], 2); + memcpy(pBuf+2, two_digits[t->year%100], 2); + pBuf[4] = '-'; + memcpy(pBuf+5, two_digits[(int)t->month], 2); + pBuf[7] = '-'; + memcpy(pBuf+8, two_digits[(int)t->day], 3); break; case NOW_YEAR: - snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d", t.year); + memcpy(pBuf, two_digits[t->year/100], 2); + memcpy(pBuf+2, two_digits[t->year%100], 3); break; case NOW_MONTH: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.month); + memcpy(pBuf, two_digits[(int)t->month], 3); break; case NOW_DAY: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.day); + memcpy(pBuf, two_digits[(int)t->day], 3); break; case NOW_HOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.hour); + memcpy(pBuf, two_digits[(int)t->hour], 3); break; case NOW_HHOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute / 30); + memcpy(pBuf, two_digits[t->minute/30], 3); break; case NOW_QHOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute / 15); + memcpy(pBuf, two_digits[t->minute/15], 3); break; case NOW_MINUTE: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute); + memcpy(pBuf, two_digits[(int)t->minute], 3); break; } @@ -2293,39 +2510,77 @@ static uchar *getNOW(eNOWType eNow) #undef tmpBUFSIZE /* clean up */ -/* Get a CEE-Property from libee. This function probably should be - * placed somewhere else, but this smells like a big restructuring - * useful in any case. So for the time being, I'll simply leave the - * function here, as the context seems good enough. -- rgerhards, 2010-12-01 - */ -static inline void -getCEEPropVal(msg_t *pMsg, es_str_t *propName, uchar **pRes, int *buflen, unsigned short *pbMustBeFreed) +/* Get a CEE-Property as string value*/ +rsRetVal +getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, rs_size_t *buflen, unsigned short *pbMustBeFreed) { - es_str_t *str = NULL; - int r; + uchar *name = NULL; + uchar *leaf; + struct json_object *parent; + struct json_object *field; + DEFiRet; if(*pbMustBeFreed) free(*pRes); *pRes = NULL; + // TODO: mutex? + if(pM->json == NULL) goto finalize_it; - if(pMsg->event == NULL) goto finalize_it; - r = ee_getEventFieldAsString(pMsg->event, propName, &str); - - if(r != EE_OK) { - DBGPRINTF("msgGtCEEVar: libee error %d during ee_getEventFieldAsString\n", r); - FINALIZE; + if(!es_strbufcmp(propName, (uchar*)"!", 1)) { + field = pM->json; + } else { + name = (uchar*)es_str2cstr(propName, NULL); + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + field = json_object_object_get(parent, (char*)leaf); + } + if(field != NULL) { + *pRes = (uchar*) strdup(json_object_get_string(field)); + *buflen = (int) ustrlen(*pRes); + *pbMustBeFreed = 1; } - *pRes = (unsigned char*) es_str2cstr(str, "#000"); - es_deleteStr(str); - *buflen = (int) ustrlen(*pRes); - *pbMustBeFreed = 1; finalize_it: + free(name); if(*pRes == NULL) { /* could not find any value, so set it to empty */ *pRes = (unsigned char*)""; *pbMustBeFreed = 0; } + RETiRet; +} + + +/* Get a CEE-Property as native json object + */ +rsRetVal +msgGetCEEPropJSON(msg_t *pM, es_str_t *propName, struct json_object **pjson) +{ + uchar *name = NULL; + uchar *leaf; + struct json_object *parent; + DEFiRet; + + // TODO: mutex? + if(pM->json == NULL) { + ABORT_FINALIZE(RS_RET_NOT_FOUND); + } + + if(!es_strbufcmp(propName, (uchar*)"!", 1)) { + *pjson = pM->json; + FINALIZE; + } + name = (uchar*)es_str2cstr(propName, NULL); + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + *pjson = json_object_object_get(parent, (char*)leaf); + if(*pjson == NULL) { + ABORT_FINALIZE(RS_RET_NOT_FOUND); + } + +finalize_it: + free(name); + RETiRet; } @@ -2513,7 +2768,7 @@ finalize_it: * Parameter "bMustBeFreed" is set by this function. It tells the * caller whether or not the string returned must be freed by the * caller itself. It is is 0, the caller MUST NOT free it. If it is - * 1, the caller MUST free 1. Handling this wrongly leads to either + * 1, the caller MUST free it. Handling this wrongly leads to either * a memory leak of a program abort (do to double-frees or frees on * the constant memory pool). So be careful to do it right. * rgerhards 2004-11-23 @@ -2529,16 +2784,16 @@ finalize_it: *pPropLen = sizeof("**OUT OF MEMORY**") - 1; \ return(UCHAR_CONSTANT("**OUT OF MEMORY**"));} uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - propid_t propid, es_str_t *propName, size_t *pPropLen, - unsigned short *pbMustBeFreed) + propid_t propid, es_str_t *propName, rs_size_t *pPropLen, + unsigned short *pbMustBeFreed, struct syslogTime *ttNow) { uchar *pRes; /* result pointer */ - int bufLen = -1; /* length of string or -1, if not known */ + rs_size_t bufLen = -1; /* length of string or -1, if not known */ uchar *pBufStart; uchar *pBuf; int iLen; short iOffs; - es_str_t *str; /* for CEE handling, temp. string */ + enum tplFormatTypes datefmt; BEGINfunc assert(pMsg != NULL); @@ -2558,7 +2813,11 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, bufLen = getMSGLen(pMsg); break; case PROP_TIMESTAMP: - pRes = (uchar*)getTimeReported(pMsg, pTpe->data.field.eDateFormat); + if (pTpe != NULL) + datefmt = pTpe->data.field.eDateFormat; + else + datefmt = tplFmtDefault; + pRes = (uchar*)getTimeReported(pMsg, datefmt); break; case PROP_HOSTNAME: pRes = (uchar*)getHOSTNAME(pMsg); @@ -2588,7 +2847,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - pRes = (uchar*)textpri((char*)pBuf, 20, getPRIi(pMsg)); + pRes = (uchar*)textpri((char*)pBuf, getPRIi(pMsg)); } break; case PROP_IUT: @@ -2608,7 +2867,11 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (uchar*)getSeverityStr(pMsg); break; case PROP_TIMEGENERATED: - pRes = (uchar*)getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); + if (pTpe != NULL) + datefmt = pTpe->data.field.eDateFormat; + else + datefmt = tplFmtDefault; + pRes = (uchar*)getTimeGenerated(pMsg, datefmt); break; case PROP_PROGRAMNAME: pRes = getProgramName(pMsg, LOCK_MUTEX); @@ -2628,74 +2891,91 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, case PROP_MSGID: pRes = (uchar*)getMSGID(pMsg); break; +#ifdef USE_LIBUUID case PROP_UUID: getUUID(pMsg, &pRes, &bufLen); break; +#endif case PROP_PARSESUCCESS: pRes = (uchar*)getParseSuccess(pMsg); break; case PROP_SYS_NOW: - if((pRes = getNOW(NOW_NOW)) == NULL) { + if((pRes = getNOW(NOW_NOW, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 10; + } break; case PROP_SYS_YEAR: - if((pRes = getNOW(NOW_YEAR)) == NULL) { + if((pRes = getNOW(NOW_YEAR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 4; + } break; case PROP_SYS_MONTH: - if((pRes = getNOW(NOW_MONTH)) == NULL) { + if((pRes = getNOW(NOW_MONTH, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_DAY: - if((pRes = getNOW(NOW_DAY)) == NULL) { + if((pRes = getNOW(NOW_DAY, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_HOUR: - if((pRes = getNOW(NOW_HOUR)) == NULL) { + if((pRes = getNOW(NOW_HOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_HHOUR: - if((pRes = getNOW(NOW_HHOUR)) == NULL) { + if((pRes = getNOW(NOW_HHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_QHOUR: - if((pRes = getNOW(NOW_QHOUR)) == NULL) { + if((pRes = getNOW(NOW_QHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_MINUTE: - if((pRes = getNOW(NOW_MINUTE)) == NULL) { + if((pRes = getNOW(NOW_MINUTE, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_MYHOSTNAME: pRes = glbl.GetLocalHostName(); break; case PROP_CEE_ALL_JSON: - if(pMsg->event == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - pRes = (uchar*) "{}"; - *pbMustBeFreed = 0; + if(pMsg->json == NULL) { + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (uchar*) "{}"; + bufLen = 2; + *pbMustBeFreed = 0; } else { - ee_fmtEventToJSON(pMsg->event, &str); - pRes = (uchar*) es_str2cstr(str, "#000"); - es_deleteStr(str); - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + pRes = (uchar*)strdup(json_object_get_string(pMsg->json)); + *pbMustBeFreed = 1; } break; case PROP_CEE: @@ -2708,13 +2988,40 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pbMustBeFreed = 0; break; case PROP_SYS_UPTIME: -# ifdef OS_SOLARIS - pRes = (uchar*) "UPTIME NOT available under Solaris"; +# ifndef HAVE_SYSINFO_UPTIME + /* An alternative on some systems (eg Solaris) is to scan + * /var/adm/utmpx for last boot time. + */ + pRes = (uchar*) "UPTIME NOT available on this system"; *pbMustBeFreed = 0; + +# elif defined(__FreeBSD__) + + { + struct timespec tp; + + if(*pbMustBeFreed == 1) + free(pRes); + if((pRes = (uchar*) MALLOC(sizeof(uchar) * 32)) == NULL) { + RET_OUT_OF_MEMORY; + } + *pbMustBeFreed = 1; + + if(clock_gettime(CLOCK_UPTIME, &tp) == -1) { + *pPropLen = sizeof("**SYSCALL FAILED**") - 1; + return(UCHAR_CONSTANT("**SYSCALL FAILED**")); + } + + snprintf((char*) pRes, sizeof(uchar) * 32, "%ld", tp.tv_sec); + } + # else + { struct sysinfo s_info; + if(*pbMustBeFreed == 1) + free(pRes); if((pRes = (uchar*) MALLOC(sizeof(uchar) * 32)) == NULL) { RET_OUT_OF_MEMORY; } @@ -2740,7 +3047,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } /* If we did not receive a template pointer, we are already done... */ - if(pTpe == NULL) { + if(pTpe == NULL || !pTpe->bComplexProcessing) { *pPropLen = (bufLen == -1) ? ustrlen(pRes) : bufLen; return pRes; } @@ -2947,13 +3254,18 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, uchar *pSb; iFrom = pTpe->data.field.iFromPos; iTo = pTpe->data.field.iToPos; - /* need to zero-base to and from (they are 1-based!) */ - if(iFrom > 0) - --iFrom; - if(iTo > 0) - --iTo; if(bufLen == -1) bufLen = ustrlen(pRes); + if(pTpe->data.field.options.bFromPosEndRelative) { + iFrom = (bufLen < iFrom) ? 0 : bufLen - iFrom; + iTo = (bufLen < iTo)? 0 : bufLen - iTo; + } else { + /* need to zero-base to and from (they are 1-based!) */ + if(iFrom > 0) + --iFrom; + if(iTo > 0) + --iTo; + } if(iFrom == 0 && iTo >= bufLen) { /* in this case, the requested string is a superset of what we already have, * so there is no need to do any processing. This is a frequent case for size-limited @@ -2962,6 +3274,8 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ ; /*DO NOTHING*/ } else { + if(iTo > bufLen) /* iTo is very large, if no to-position is set in the template! */ + iTo = bufLen; iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); if(pBuf == NULL) { @@ -3327,9 +3641,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, jsonField(pTpe, &pRes, pbMustBeFreed, &bufLen); } - if(bufLen == -1) - bufLen = ustrlen(pRes); - *pPropLen = bufLen; + *pPropLen = (bufLen == -1) ? ustrlen(pRes) : bufLen; ENDfunc return(pRes); @@ -3346,29 +3658,25 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, es_str_t* msgGetCEEVarNew(msg_t *pMsg, char *name) { + uchar *leaf; + char *val; es_str_t *estr = NULL; - es_str_t *epropName = NULL; - struct ee_field *field; + struct json_object *json, *parent; ISOBJ_TYPE_assert(pMsg, msg); - if(pMsg->event == NULL) { + if(pMsg->json == NULL) { estr = es_newStr(1); goto done; } - - epropName = es_newStrFromCStr(name, strlen(name)); // TODO: optimize (in grammar!) - field = ee_getEventField(pMsg->event, epropName); - if(field != NULL) { - ee_getFieldAsString(field, &estr); - } - if(estr == NULL) { - DBGPRINTF("msgGetCEEVar: error obtaining var (field=%p, var='%s')\n", - field, name); - estr = es_newStrFromCStr("*ERROR*", sizeof("*ERROR*") - 1); + leaf = jsonPathGetLeaf((uchar*)name, strlen(name)); + if(jsonPathFindParent(pMsg, (uchar*)name, leaf, &parent, 1) != RS_RET_OK) { + estr = es_newStr(1); + goto done; } - es_deleteStr(epropName); - + json = json_object_object_get(parent, (char*)leaf); + val = (char*)json_object_get_string(json); + estr = es_newStrFromCStr(val, strlen(val)); done: return estr; } @@ -3379,7 +3687,7 @@ done: es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name) { - size_t propLen; + rs_size_t propLen; uchar *pszProp = NULL; propid_t propid; unsigned short bMustBeFreed = 0; @@ -3390,7 +3698,7 @@ msgGetMsgVarNew(msg_t *pThis, uchar *name) /* always call MsgGetProp() without a template specifier */ /* TODO: optimize propNameToID() call -- rgerhards, 2009-06-26 */ propNameStrToID(name, &propid); - pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed); + pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed, NULL); estr = es_newStrFromCStr((char*)pszProp, propLen); if(bMustBeFreed) @@ -3412,6 +3720,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) prop_t *myProp; prop_t *propRcvFrom = NULL; prop_t *propRcvFromIP = NULL; + struct json_tokener *tokener; + struct json_object *json; DEFiRet; ISOBJ_TYPE_assert(pThis, msg); @@ -3466,6 +3776,12 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) MsgSetRulesetByName(pThis, pProp->val.pStr); } else if(isProp("pszMSG")) { dbgprintf("no longer supported property pszMSG silently ignored\n"); + } else if(isProp("json")) { + tokener = json_tokener_new(); + json = json_tokener_parse_ex(tokener, (char*)rsCStrGetSzStrNoNULL(pProp->val.pStr), + cstrLen(pProp->val.pStr)); + json_tokener_free(tokener); + msgAddJSON(pThis, (uchar*)"!", json); } else { dbgprintf("unknown supported property '%s' silently ignored\n", rsCStrGetSzStrNoNULL(pProp->pcsName)); @@ -3481,16 +3797,293 @@ finalize_it: * satisfies the base object class getSeverity semantics. * rgerhards, 2008-01-14 */ -static rsRetVal -MsgGetSeverity(obj_t_ptr pThis, int *piSeverity) +rsRetVal +MsgGetSeverity(msg_t *pMsg, int *piSeverity) { - ISOBJ_TYPE_assert(pThis, msg); - assert(piSeverity != NULL); - *piSeverity = ((msg_t*) pThis)->iSeverity; + *piSeverity = pMsg->iSeverity; return RS_RET_OK; } +static uchar * +jsonPathGetLeaf(uchar *name, int lenName) +{ + int i; + for(i = lenName ; name[i] != '!' && i >= 0 ; --i) + /* just skip */; + if(name[i] == '!') + ++i; + return name + i; +} + + +static rsRetVal +jsonPathFindNext(struct json_object *root, uchar **name, uchar *leaf, + struct json_object **found, int bCreate) +{ + uchar namebuf[1024]; + struct json_object *json; + size_t i; + uchar *p = *name; + DEFiRet; + + if(*p == '!') + ++p; + for(i = 0 ; *p && *p != '!' && p != leaf && i < sizeof(namebuf)-1 ; ++i, ++p) + namebuf[i] = *p; + if(i > 0) { + namebuf[i] = '\0'; + dbgprintf("AAAA: next JSONPath elt: '%s'\n", namebuf); + json = json_object_object_get(root, (char*)namebuf); + } else + json = root; + if(json == NULL) { + if(!bCreate) { + ABORT_FINALIZE(RS_RET_JNAME_INVALID); + } else { + json = json_object_new_object(); + json_object_object_add(root, (char*)namebuf, json); + } + } + + *name = p; + *found = json; +finalize_it: + RETiRet; +} + +static rsRetVal +jsonPathFindParent(msg_t *pM, uchar *name, uchar *leaf, struct json_object **parent, int bCreate) +{ + DEFiRet; + *parent = pM->json; + while(name < leaf-1) { + jsonPathFindNext(*parent, &name, leaf, parent, bCreate); + } + RETiRet; +} + +static rsRetVal +jsonMerge(struct json_object *existing, struct json_object *json) +{ + /* TODO: check & handle duplicate names */ + DEFiRet; + struct json_object_iter it; + + json_object_object_foreachC(json, it) { +DBGPRINTF("AAAA jsonMerge adds '%s'\n", it.key); + json_object_object_add(existing, it.key, + json_object_get(it.val)); + } + /* note: json-c does ref counting. We added all descandants refcounts + * in the loop above. So when we now free(_put) the root object, only + * root gets freed(). + */ + json_object_put(json); + RETiRet; +} + +/* find a JSON structure element (field or container doesn't matter). */ +rsRetVal +jsonFind(msg_t *pM, es_str_t *propName, struct json_object **jsonres) +{ + uchar *name = NULL; + uchar *leaf; + struct json_object *parent; + struct json_object *field; + DEFiRet; + + if(pM->json == NULL) { + field = NULL; + goto finalize_it; + } + + if(!es_strbufcmp(propName, (uchar*)"!", 1)) { + field = pM->json; + } else { + name = (uchar*)es_str2cstr(propName, NULL); + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 0)); + field = json_object_object_get(parent, (char*)leaf); + } + *jsonres = field; + +finalize_it: + free(name); + RETiRet; +} + +rsRetVal +msgAddJSON(msg_t *pM, uchar *name, struct json_object *json) +{ + /* TODO: error checks! This is a quick&dirty PoC! */ + struct json_object *parent, *leafnode; + uchar *leaf; + DEFiRet; + + MsgLock(pM); + if(name[0] == '!' && name[1] == '\0') { + if(pM->json == NULL) + pM->json = json; + else + CHKiRet(jsonMerge(pM->json, json)); + } else { + if(pM->json == NULL) { + /* now we need a root obj */ + pM->json = json_object_new_object(); + } + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + leafnode = json_object_object_get(parent, (char*)leaf); + if(leafnode == NULL) { + json_object_object_add(parent, (char*)leaf, json); + } else { + if(json_object_get_type(json) == json_type_object) { + CHKiRet(jsonMerge(pM->json, json)); + } else { +//dbgprintf("AAAA: leafnode already exists, type is %d, update with %d\n", (int)json_object_get_type(leafnode), (int)json_object_get_type(json)); + /* TODO: improve the code below, however, the current + * state is not really bad */ + if(json_object_get_type(leafnode) == json_type_object) { + DBGPRINTF("msgAddJSON: trying to update a container " + "node with a leaf, name is '%s' - " + "forbidden\n", name); + json_object_put(json); + ABORT_FINALIZE(RS_RET_INVLD_SETOP); + } + /* json-c code indicates we can simply replace a + * json type. Unfortunaltely, this is not documented + * as part of the interface spec. We still use it, + * because it speeds up processing. If it does not work + * at some point, use + * json_object_object_del(parent, (char*)leaf); + * before adding. rgerhards, 2012-09-17 + */ + json_object_object_add(parent, (char*)leaf, json); + } + } + } + +finalize_it: + MsgUnlock(pM); + RETiRet; +} + +rsRetVal +msgDelJSON(msg_t *pM, uchar *name) +{ + struct json_object *parent, *leafnode; + uchar *leaf; + DEFiRet; + +dbgprintf("AAAA: unset variable '%s'\n", name); + MsgLock(pM); + if(name[0] == '!' && name[1] == '\0') { + /* strange, but I think we should permit this. After all, + * we trust rsyslog.conf to be written by the admin. + */ + DBGPRINTF("unsetting JSON root object\n"); + json_object_put(pM->json); + pM->json = NULL; + } else { + if(pM->json == NULL) { + /* now we need a root obj */ + pM->json = json_object_new_object(); + } + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + leafnode = json_object_object_get(parent, (char*)leaf); +DBGPRINTF("AAAA: unset found JSON value path '%s', " "leaf '%s', leafnode %p\n", name, leaf, leafnode); + if(leafnode == NULL) { + DBGPRINTF("unset JSON: could not find '%s'\n", name); + ABORT_FINALIZE(RS_RET_JNAME_NOTFOUND); + } else { + DBGPRINTF("deleting JSON value path '%s', " + "leaf '%s', type %d\n", + name, leaf, json_object_get_type(leafnode)); + json_object_object_del(parent, (char*)leaf); + } + } + +finalize_it: + MsgUnlock(pM); + RETiRet; +} + +static struct json_object * +jsonDeepCopy(struct json_object *src) +{ + struct json_object *dst = NULL, *json; + struct json_object_iter it; + int arrayLen, i; + + if(src == NULL) goto done; + + switch(json_object_get_type(src)) { + case json_type_boolean: + dst = json_object_new_boolean(json_object_get_boolean(src)); + break; + case json_type_double: + dst = json_object_new_double(json_object_get_double(src)); + break; + case json_type_int: + dst = json_object_new_int(json_object_get_int(src)); + break; + case json_type_string: + dst = json_object_new_string(json_object_get_string(src)); + break; + case json_type_object: + dst = json_object_new_object(); + json_object_object_foreachC(src, it) { + json = jsonDeepCopy(it.val); + json_object_object_add(dst, it.key, json); + } + break; + case json_type_array: + arrayLen = json_object_array_length(src); + dst = json_object_new_array(); + for(i = 0 ; i < arrayLen ; ++i) { + json = json_object_array_get_idx(src, i); + json = jsonDeepCopy(json); + json_object_array_add(dst, json); + } + break; + default:DBGPRINTF("jsonDeepCopy(): error unknown type %d\n", + json_object_get_type(src)); + dst = NULL; + break; + } +done: return dst; +} + + +rsRetVal +msgSetJSONFromVar(msg_t *pMsg, uchar *varname, struct var *v) +{ + struct json_object *json = NULL; + char *cstr; + DEFiRet; + switch(v->datatype) { + case 'S':/* string */ + cstr = es_str2cstr(v->d.estr, NULL); + json = json_object_new_string(cstr); + free(cstr); + break; + case 'N':/* number (integer) */ + json = json_object_new_int((int) v->d.n); + break; + case 'J':/* native JSON */ + json = jsonDeepCopy(v->d.json); + break; + default:DBGPRINTF("msgSetJSONFromVar: unsupported datatype %c\n", + v->datatype); + ABORT_FINALIZE(RS_RET_ERR); + } + msgAddJSON(pMsg, varname+1, json); +finalize_it: + RETiRet; +} + /* dummy */ rsRetVal msgQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } @@ -3503,11 +4096,10 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(var, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_SERIALIZE, MsgSerialize); - OBJSetMethodHandler(objMethod_SETPROPERTY, MsgSetProperty); - OBJSetMethodHandler(objMethod_GETSEVERITY, MsgGetSeverity); /* some more inits */ # if HAVE_MALLOC_TRIM INIT_ATOMIC_HELPER_MUT(mutTrimCtr); diff --git a/runtime/msg.h b/runtime/msg.h index c0b50709..6faf066a 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -3,7 +3,7 @@ * * File begun on 2007-07-13 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -30,12 +30,11 @@ #include <pthread.h> #include <libestr.h> +#include <json/json.h> #include "obj.h" #include "syslogd-types.h" #include "template.h" #include "atomic.h" -#include "libee/libee.h" - /* rgerhards 2004-11-08: The following structure represents a * syslog message. @@ -75,6 +74,7 @@ struct msg { int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ int iLenHOSTNAME; /* Length of HOSTNAME */ + int iLenPROGNAME; /* Length of PROGNAME (-1 = not yet set) */ uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we * need to preserve cryptographic verifiers. */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ @@ -86,7 +86,6 @@ struct msg { char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ - cstr_t *pCSProgName; /* the (BSD) program name */ cstr_t *pCSStrucData; /* STRUCTURED-DATA */ cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ @@ -108,11 +107,15 @@ struct msg { it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ - struct ee_event *event; /**< libee event */ + struct json_object *json; /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE]; union { + uchar *ptr; /* pointer to progname value */ + uchar szBuf[CONF_PROGNAME_BUFSIZE]; + } PROGNAME; + union { uchar *pszTAG; /* pointer to tag value */ uchar szBuf[CONF_TAG_BUFSIZE]; } TAG; @@ -145,6 +148,8 @@ struct msg { PROTOTYPEObjClassInit(msg); rsRetVal msgConstruct(msg_t **ppThis); rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); +rsRetVal msgConstructForDeserializer(msg_t **ppThis); +rsRetVal msgConstructFinalizer(msg_t *pThis); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); @@ -171,8 +176,7 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, propid_t propid, es_str_t *propName, - size_t *pPropLen, unsigned short *pbMustBeFreed); -char *textpri(char *pRes, size_t pResLen, int pri); + rs_size_t *pPropLen, unsigned short *pbMustBeFreed, struct syslogTime *ttNow); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name); uchar *getRcvFrom(msg_t *pM); @@ -182,21 +186,34 @@ char *getPRI(msg_t *pMsg); void getRawMsg(msg_t *pM, uchar **pBuf, int *piLen); rsRetVal msgGetCEEVar(msg_t *pThis, cstr_t *propName, var_t **ppVar); es_str_t* msgGetCEEVarNew(msg_t *pMsg, char *name); +rsRetVal msgAddJSON(msg_t *pM, uchar *name, struct json_object *json); +rsRetVal getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, rs_size_t *buflen, unsigned short *pbMustBeFreed); +rsRetVal MsgGetSeverity(msg_t *pThis, int *piSeverity); +rsRetVal MsgDeserialize(msg_t *pMsg, strm_t *pStrm); /* TODO: remove these five (so far used in action.c) */ uchar *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); char *getPROCID(msg_t *pM, sbool bLockMutex); char *getAPPNAME(msg_t *pM, sbool bLockMutex); +void setMSGLen(msg_t *pM, int lenMsg); int getMSGLen(msg_t *pM); char *getHOSTNAME(msg_t *pM); int getHOSTNAMELen(msg_t *pM); uchar *getProgramName(msg_t *pM, sbool bLockMutex); -int getProgramNameLen(msg_t *pM, sbool bLockMutex); uchar *getRcvFrom(msg_t *pM); rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); uchar *propIDToName(propid_t propID); +rsRetVal msgGetCEEPropJSON(msg_t *pM, es_str_t *propName, struct json_object **pjson); +rsRetVal msgSetJSONFromVar(msg_t *pMsg, uchar *varname, struct var *var); +rsRetVal msgDelJSON(msg_t *pMsg, uchar *varname); +rsRetVal jsonFind(msg_t *pM, es_str_t *propName, struct json_object **jsonres); + +static inline rsRetVal +msgUnsetJSON(msg_t *pMsg, uchar *varname) { + return msgDelJSON(pMsg, varname+1); +} /* ------------------------------ some inline functions ------------------------------ */ diff --git a/runtime/net.c b/runtime/net.c index dcf9cb52..13391cc0 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -54,7 +54,11 @@ #include <fnmatch.h> #include <fcntl.h> #include <unistd.h> +#if HAVE_GETIFADDRS #include <ifaddrs.h> +#else +#include "compat/ifaddrs.h" +#endif /* HAVE_GETIFADDRS */ #include <sys/types.h> #include <arpa/inet.h> @@ -66,6 +70,7 @@ #include "errmsg.h" #include "net.h" #include "dnscache.h" +#include "prop.h" #ifdef OS_SOLARIS # define s6_addr32 _S6_un._S6_u32 @@ -79,6 +84,7 @@ MODULE_TYPE_NOKEEP DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) /* support for defining allowed TCP and UDP senders. We use the same * structure to implement this (a linked list), but we define two different @@ -226,6 +232,7 @@ finalize_it: /* enqueue the element */ if(pPeer->pWildcardRoot == NULL) { pPeer->pWildcardRoot = pNew; + pPeer->pWildcardLast = pNew; } else { pPeer->pWildcardLast->pNext = pNew; } @@ -575,7 +582,7 @@ static void clearAllowedSenders(uchar *pszType) { struct AllowedSenders *pPrev; - struct AllowedSenders *pCurr; + struct AllowedSenders *pCurr = NULL; if(setAllowRoot(&pCurr, pszType) != RS_RET_OK) return; /* if something went wrong, so let's leave */ @@ -983,7 +990,7 @@ MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { struct AllowedSenders *pAllow; - struct AllowedSenders *pAllowRoot; + struct AllowedSenders *pAllowRoot = NULL; int bNeededDNS = 0; /* partial check because we could not resolve DNS? */ int ret; @@ -1111,98 +1118,15 @@ void debugListenInfo(int fd, char *type) } -/* Return a printable representation of a host address. - * Now (2007-07-16) also returns the full host name (if it could be obtained) - * in the second param [thanks to mildew@gmail.com for the patch]. - * The caller must provide buffer space for pszHost and pszHostFQDN. These - * buffers must be of size NI_MAXHOST. This is not checked here, because - * there is no way to check it. We use this way of doing things because it - * frees us from using dynamic memory allocation where it really does not - * pay. - * 2005-05-16 rgerhards: added IP representation. Must also be NI_MAXHOST +/* Return a printable representation of a host addresses. If + * a parameter is NULL, it is not set. rgerhards, 2013-01-22 */ -rsRetVal cvthname(struct sockaddr_storage *f, uchar *pszHost, uchar *pszHostFQDN, uchar *pszIP) +rsRetVal +cvthname(struct sockaddr_storage *f, prop_t **localName, prop_t **fqdn, prop_t **ip) { DEFiRet; - register uchar *p; - int count; - assert(f != NULL); - assert(pszHost != NULL); - assert(pszHostFQDN != NULL); - - iRet = dnscacheLookup(f, pszHostFQDN, pszIP); - - if(iRet == RS_RET_INVALID_SOURCE) { - strcpy((char*) pszHost, (char*) pszHostFQDN); /* we use whatever was provided as replacement */ - ABORT_FINALIZE(RS_RET_OK); /* this is handled, we are happy with it */ - } else if(iRet != RS_RET_OK) { - FINALIZE; /* we return whatever error state we have - can not handle it */ - } - - /* if we reach this point, we obtained a non-numeric hostname and can now process it */ - - /* Convert to lower case */ - for(p = pszHostFQDN ; *p ; p++) - if (isupper((int) *p)) - *p = tolower(*p); - - /* OK, the fqdn is now known. Now it is time to extract only the hostname - * part if we were instructed to do so. - */ - /* TODO: quick and dirty right now: we need to optimize that. We simply - * copy over the buffer and then use the old code. In the long term, that should - * be placed in its own function and probably outside of the net module (at least - * if should no longer reley on syslogd.c's global config-setting variables). - * Note that the old code always removes the local domain. We may want to - * make this in option in the long term. (rgerhards, 2007-09-11) - */ - strcpy((char*)pszHost, (char*)pszHostFQDN); - if( (glbl.GetPreserveFQDN() == 0) - && (p = (uchar*) strchr((char*)pszHost, '.'))) { /* find start of domain name "machine.example.com" */ - strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()); - if(strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()) == 0) { - *p = '\0'; /* simply terminate the string */ - } else { - /* now check if we belong to any of the domain names that were specified - * in the -s command line option. If so, remove and we are done. - * TODO: this must go away! -- rgerhards, 2008-04-16 - * For proper modularization, this must be done different, e.g. via a - * "to be stripped" property of *this* object itself. - */ - if(glbl.GetStripDomains() != NULL) { - count=0; - while(glbl.GetStripDomains()[count]) { - if (strcmp((char*)(p + 1), glbl.GetStripDomains()[count]) == 0) { - *p = '\0'; - FINALIZE; /* we are done */ - } - count++; - } - } - /* if we reach this point, we have not found any domain we should strip. Now - * we try and see if the host itself is listed in the -l command line option - * and so should be stripped also. If so, we do it and return. Please note that - * -l list FQDNs, not just the hostname part. If it did just list the hostname, the - * door would be wide-open for all kinds of mixing up of hosts. Because of this, - * you'll see comparison against the full string (pszHost) below. The termination - * still occurs at *p, which points at the first dot after the hostname. - * TODO: this must also go away - see comment above -- rgerhards, 2008-04-16 - */ - if(glbl.GetLocalHosts() != NULL) { - count=0; - while (glbl.GetLocalHosts()[count]) { - if (!strcmp((char*)pszHost, (char*)glbl.GetLocalHosts()[count])) { - *p = '\0'; - break; /* we are done */ - } - count++; - } - } - } - } - -finalize_it: + iRet = dnscacheLookup(f, NULL, fqdn, localName, ip); RETiRet; } @@ -1467,7 +1391,7 @@ finalize_it: */ static rsRetVal HasRestrictions(uchar *pszType, int *bHasRestrictions) { - struct AllowedSenders *pAllowRoot; + struct AllowedSenders *pAllowRoot = NULL; DEFiRet; CHKiRet(setAllowRoot(&pAllowRoot, pszType)); @@ -1577,6 +1501,7 @@ BEGINObjClassExit(net, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO CODESTARTObjClassExit(net) /* release objects we no longer need */ objRelease(glbl, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); ENDObjClassExit(net) @@ -1589,6 +1514,7 @@ BEGINAbstractObjClassInit(net, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ ENDObjClassInit(net) diff --git a/runtime/net.h b/runtime/net.h index 1b41c81c..b196116b 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -1,6 +1,6 @@ /* Definitions for network-related stuff. * - * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -131,7 +131,7 @@ struct permittedPeers_s { /* interfaces */ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ - rsRetVal (*cvthname)(struct sockaddr_storage *f, uchar *pszHost, uchar *pszHostFQDN, uchar *pszIP); + rsRetVal (*cvthname)(struct sockaddr_storage *f, prop_t **localName, prop_t **fqdn, prop_t **ip); /* things to go away after proper modularization */ rsRetVal (*addAllowedSenderLine)(char* pName, uchar** ppRestOfConfLine); void (*PrintAllowedSenders)(int iListToPrint); @@ -156,8 +156,9 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ /* data members - these should go away over time... TODO */ int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ + /* v8 cvthname() signature change -- rgerhards, 2013-01-18 */ ENDinterface(net) -#define netCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ +#define netCURR_IF_VERSION 8 /* increment whenever you change the interface structure! */ /* prototypes */ PROTOTYPEObj(net); diff --git a/runtime/netstrm.c b/runtime/netstrm.c index 58f38280..c046cf52 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -271,11 +271,11 @@ GetRemoteHName(netstrm_t *pThis, uchar **ppsz) /* get remote IP - slim wrapper for NSD driver function */ static rsRetVal -GetRemoteIP(netstrm_t *pThis, uchar **ppsz) +GetRemoteIP(netstrm_t *pThis, prop_t **ip) { DEFiRet; ISOBJ_TYPE_assert(pThis, netstrm); - iRet = pThis->Drvr.GetRemoteIP(pThis->pDrvrData, ppsz); + iRet = pThis->Drvr.GetRemoteIP(pThis->pDrvrData, ip); RETiRet; } diff --git a/runtime/netstrm.h b/runtime/netstrm.h index ee8d9e59..4ef24229 100644 --- a/runtime/netstrm.h +++ b/runtime/netstrm.h @@ -49,7 +49,7 @@ BEGINinterface(netstrm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Send)(netstrm_t *pThis, uchar *pBuf, ssize_t *pLenBuf); rsRetVal (*Connect)(netstrm_t *pThis, int family, unsigned char *port, unsigned char *host); rsRetVal (*GetRemoteHName)(netstrm_t *pThis, uchar **pszName); - rsRetVal (*GetRemoteIP)(netstrm_t *pThis, uchar **pszIP); + rsRetVal (*GetRemoteIP)(netstrm_t *pThis, prop_t **ip); rsRetVal (*SetDrvrMode)(netstrm_t *pThis, int iMode); rsRetVal (*SetDrvrAuthMode)(netstrm_t *pThis, uchar*); rsRetVal (*SetDrvrPermPeers)(netstrm_t *pThis, permittedPeers_t*); @@ -72,10 +72,11 @@ BEGINinterface(netstrm) /* name must also be changed in ENDinterface macro! */ /* v4 */ rsRetVal (*EnableKeepAlive)(netstrm_t *pThis); ENDinterface(netstrm) -#define netstrmCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define netstrmCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* interface version 3 added GetRemAddr() * interface version 4 added EnableKeepAlive() -- rgerhards, 2009-06-02 * interface version 5 changed return of CheckConnection from void to rsRetVal -- alorbach, 2012-09-06 + * interface version 6 changed signature of GetRemoteIP() -- rgerhards, 2013-01-21 * */ /* prototypes */ diff --git a/runtime/nsd.h b/runtime/nsd.h index d7d6abbd..aa3662a4 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -59,7 +59,7 @@ BEGINinterface(nsd) /* name must also be changed in ENDinterface macro! */ uchar *pLstnPort, uchar *pLstnIP, int iSessMax); rsRetVal (*AcceptConnReq)(nsd_t *pThis, nsd_t **ppThis); rsRetVal (*GetRemoteHName)(nsd_t *pThis, uchar **pszName); - rsRetVal (*GetRemoteIP)(nsd_t *pThis, uchar **pszIP); + rsRetVal (*GetRemoteIP)(nsd_t *pThis, prop_t **ip); rsRetVal (*SetMode)(nsd_t *pThis, int mode); /* sets a driver specific mode - see driver doc for details */ rsRetVal (*SetAuthMode)(nsd_t *pThis, uchar*); /* sets a driver specific mode - see driver doc for details */ rsRetVal (*SetPermPeers)(nsd_t *pThis, permittedPeers_t*); /* sets driver permitted peers for auth needs */ @@ -80,10 +80,11 @@ BEGINinterface(nsd) /* name must also be changed in ENDinterface macro! */ /* v5 */ rsRetVal (*EnableKeepAlive)(nsd_t *pThis); ENDinterface(nsd) -#define nsdCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ +#define nsdCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* interface version 4 added GetRemAddr() * interface version 5 added EnableKeepAlive() -- rgerhards, 2009-06-02 * interface version 6 changed return of CheckConnection from void to rsRetVal -- alorbach, 2012-09-06 + * interface version 7 changed signature ofGetRempoteIP() -- rgerhards, 2013-01-21 */ /* interface for the select call */ diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 71eafbd2..6ef4feba 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -259,9 +259,9 @@ gtlsClientCertCallback(gnutls_session session, static rsRetVal gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) { - char dn[128]; - uchar lnBuf[256]; - size_t size; + uchar szBufA[1024]; + uchar *szBuf = szBufA; + size_t szBufLen = sizeof(szBufA), tmp; unsigned int algo, bits; time_t expiration_time, activation_time; const gnutls_datum *cert_list; @@ -271,8 +271,6 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) int gnuRet; DEFiRet; unsigned iAltName; - size_t szAltNameLen; - char szAltName[1024]; /* this is sufficient for the DNSNAME... */ assert(ppStr != NULL); ISOBJ_TYPE_assert(pThis, nsd_gtls); @@ -281,61 +279,62 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) return RS_RET_TLS_CERT_ERR; cert_list = gnutls_certificate_get_peers(pThis->sess, &cert_list_size); - - CHKiRet(rsCStrConstruct(&pStr)); - - snprintf((char*)lnBuf, sizeof(lnBuf), "peer provided %d certificate(s). ", cert_list_size); - CHKiRet(rsCStrAppendStr(pStr, lnBuf)); + CHKiRet(rsCStrConstructFromszStrf(&pStr, "peer provided %d certificate(s). ", cert_list_size)); if(cert_list_size > 0) { /* we only print information about the first certificate */ CHKgnutls(gnutls_x509_crt_init(&cert)); CHKgnutls(gnutls_x509_crt_import(cert, &cert_list[0], GNUTLS_X509_FMT_DER)); - CHKiRet(rsCStrAppendStr(pStr, (uchar*)"Certificate 1 info: ")); - expiration_time = gnutls_x509_crt_get_expiration_time(cert); activation_time = gnutls_x509_crt_get_activation_time(cert); - ctime_r(&activation_time, dn); - dn[strlen(dn) - 1] = '\0'; /* strip linefeed */ - snprintf((char*)lnBuf, sizeof(lnBuf), "certificate valid from %s ", dn); - CHKiRet(rsCStrAppendStr(pStr, lnBuf)); - - ctime_r(&expiration_time, dn); - dn[strlen(dn) - 1] = '\0'; /* strip linefeed */ - snprintf((char*)lnBuf, sizeof(lnBuf), "to %s; ", dn); - CHKiRet(rsCStrAppendStr(pStr, lnBuf)); + ctime_r(&activation_time, szBuf); + szBuf[strlen(szBuf) - 1] = '\0'; /* strip linefeed */ + CHKiRet(rsCStrAppendStrf(pStr, (uchar*)"Certificate 1 info: " + "certificate valid from %s ", szBuf)); + ctime_r(&expiration_time, szBuf); + szBuf[strlen(szBuf) - 1] = '\0'; /* strip linefeed */ + CHKiRet(rsCStrAppendStrf(pStr, "to %s; ", szBuf)); /* Extract some of the public key algorithm's parameters */ algo = gnutls_x509_crt_get_pk_algorithm(cert, &bits); - - snprintf((char*)lnBuf, sizeof(lnBuf), "Certificate public key: %s; ", - gnutls_pk_algorithm_get_name(algo)); - CHKiRet(rsCStrAppendStr(pStr, lnBuf)); + CHKiRet(rsCStrAppendStrf(pStr, "Certificate public key: %s; ", + gnutls_pk_algorithm_get_name(algo))); /* names */ - size = sizeof(dn); - gnutls_x509_crt_get_dn(cert, dn, &size); - snprintf((char*)lnBuf, sizeof(lnBuf), "DN: %s; ", dn); - CHKiRet(rsCStrAppendStr(pStr, lnBuf)); - - size = sizeof(dn); - gnutls_x509_crt_get_issuer_dn(cert, dn, &size); - snprintf((char*)lnBuf, sizeof(lnBuf), "Issuer DN: %s; ", dn); - CHKiRet(rsCStrAppendStr(pStr, lnBuf)); + tmp = szBufLen; + if(gnutls_x509_crt_get_dn(cert, szBuf, &tmp) + == GNUTLS_E_SHORT_MEMORY_BUFFER) { + szBufLen = tmp; + szBuf = malloc(tmp); + gnutls_x509_crt_get_dn(cert, szBuf, &tmp); + } + CHKiRet(rsCStrAppendStrf(pStr, "DN: %s; ", szBuf)); + + tmp = szBufLen; + if(gnutls_x509_crt_get_issuer_dn(cert, szBuf, &tmp) + == GNUTLS_E_SHORT_MEMORY_BUFFER) { + szBufLen = tmp; + szBuf = realloc((szBuf == szBufA) ? NULL : szBuf, tmp); + gnutls_x509_crt_get_issuer_dn(cert, szBuf, &tmp); + } + CHKiRet(rsCStrAppendStrf(pStr, "Issuer DN: %s; ", szBuf)); /* dNSName alt name */ iAltName = 0; while(1) { /* loop broken below */ - szAltNameLen = sizeof(szAltName); + tmp = szBufLen; gnuRet = gnutls_x509_crt_get_subject_alt_name(cert, iAltName, - szAltName, &szAltNameLen, NULL); - if(gnuRet < 0) + szBuf, &tmp, NULL); + if(gnuRet == GNUTLS_E_SHORT_MEMORY_BUFFER) { + szBufLen = tmp; + szBuf = realloc((szBuf == szBufA) ? NULL : szBuf, tmp); + continue; + } else if(gnuRet < 0) break; else if(gnuRet == GNUTLS_SAN_DNSNAME) { /* we found it! */ - snprintf((char*)lnBuf, sizeof(lnBuf), "SAN:DNSname: %s; ", szAltName); - CHKiRet(rsCStrAppendStr(pStr, lnBuf)); + CHKiRet(rsCStrAppendStrf(pStr, "SAN:DNSname: %s; ", szBuf)); /* do NOT break, because there may be multiple dNSName's! */ } ++iAltName; @@ -352,6 +351,8 @@ finalize_it: if(pStr != NULL) rsCStrDestruct(&pStr); } + if(szBuf != szBufA) + free(szBuf); RETiRet; } @@ -1357,16 +1358,14 @@ GetRemAddr(nsd_t *pNsd, struct sockaddr_storage **ppAddr) } -/* get the remote host's IP address. The returned string must be freed by the - * caller. -- rgerhards, 2008-04-25 - */ +/* get the remote host's IP address. Caller must Destruct the object. */ static rsRetVal -GetRemoteIP(nsd_t *pNsd, uchar **ppszIP) +GetRemoteIP(nsd_t *pNsd, prop_t **ip) { DEFiRet; nsd_gtls_t *pThis = (nsd_gtls_t*) pNsd; ISOBJ_TYPE_assert(pThis, nsd_gtls); - iRet = nsd_ptcp.GetRemoteIP(pThis->pTcp, ppszIP); + iRet = nsd_ptcp.GetRemoteIP(pThis->pTcp, ip); RETiRet; } diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index 12f891ea..f889a00e 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -2,7 +2,7 @@ * * An implementation of the nsd interface for plain tcp sockets. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -50,6 +50,7 @@ #include "nsdsel_ptcp.h" #include "nsdpoll_ptcp.h" #include "nsd_ptcp.h" +#include "prop.h" #include "dnscache.h" MODULE_TYPE_LIB @@ -62,6 +63,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(net) DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) +DEFobjCurrIf(prop) /* a few deinit helpers */ @@ -87,10 +89,9 @@ ENDobjConstruct(nsd_ptcp) BEGINobjDestruct(nsd_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(nsd_ptcp) sockClose(&pThis->sock); - if(pThis->pRemHostIP != NULL) - free(pThis->pRemHostIP); - if(pThis->pRemHostName != NULL) - free(pThis->pRemHostName); + if(pThis->remoteIP != NULL) + prop.Destruct(&pThis->remoteIP); + free(pThis->pRemHostName); ENDobjDestruct(nsd_ptcp) @@ -251,32 +252,22 @@ Abort(nsd_t *pNsd) static rsRetVal FillRemHost(nsd_ptcp_t *pThis, struct sockaddr_storage *pAddr) { - uchar szIP[NI_MAXHOST] = ""; - uchar szHname[NI_MAXHOST] = ""; - size_t len; + prop_t *fqdn; DEFiRet; ISOBJ_TYPE_assert(pThis, nsd_ptcp); assert(pAddr != NULL); - CHKiRet(dnscacheLookup(pAddr, szHname, szIP)); + CHKiRet(dnscacheLookup(pAddr, &fqdn, NULL, NULL, &pThis->remoteIP)); /* We now have the names, so now let's allocate memory and store them permanently. * (side note: we may hold on to these values for quite a while, thus we trim their * memory consumption) */ - len = strlen((char*)szIP) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostIP = MALLOC(len)) == NULL) + if((pThis->pRemHostName = MALLOC(prop.GetStringLen(fqdn)+1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - memcpy(pThis->pRemHostIP, szIP, len); - - len = strlen((char*)szHname) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostName = MALLOC(len)) == NULL) { - free(pThis->pRemHostIP); /* prevent leak */ - pThis->pRemHostIP = NULL; - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - memcpy(pThis->pRemHostName, szHname, len); + memcpy(pThis->pRemHostName, propGetSzStr(fqdn), prop.GetStringLen(fqdn)+1); + prop.Destruct(&fqdn); finalize_it: RETiRet; @@ -460,7 +451,9 @@ LstnInit(netstrms_t *pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t*), #endif ) { /* TODO: check if *we* bound the socket - else we *have* an error! */ - dbgprintf("error %d while binding tcp socket\n", errno); + char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + dbgprintf("error %d while binding tcp socket: %s\n", errno, errStr); close(sock); sock = -1; continue; @@ -717,21 +710,16 @@ finalize_it: } -/* get the remote host's IP address. The returned string must be freed by the - * caller. - * rgerhards, 2008-04-24 +/* get the remote host's IP address. Caller must Destruct the object. */ static rsRetVal -GetRemoteIP(nsd_t *pNsd, uchar **ppszIP) +GetRemoteIP(nsd_t *pNsd, prop_t **ip) { DEFiRet; nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; ISOBJ_TYPE_assert(pThis, nsd_ptcp); - assert(ppszIP != NULL); - - CHKmalloc(*ppszIP = (uchar*)strdup(pThis->pRemHostIP == NULL ? "" : (char*) pThis->pRemHostIP)); - -finalize_it: + prop.AddRef(pThis->remoteIP); + *ip = pThis->remoteIP; RETiRet; } @@ -777,6 +765,7 @@ CODESTARTObjClassExit(nsd_ptcp) /* release objects we no longer need */ objRelease(net, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(netstrm, DONT_LOAD_LIB); objRelease(netstrms, LM_NETSTRMS_FILENAME); @@ -791,6 +780,7 @@ BEGINObjClassInit(nsd_ptcp, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(net, CORE_COMPONENT)); CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); diff --git a/runtime/nsd_ptcp.h b/runtime/nsd_ptcp.h index a1bcd646..ed6b8565 100644 --- a/runtime/nsd_ptcp.h +++ b/runtime/nsd_ptcp.h @@ -30,7 +30,7 @@ typedef nsd_if_t nsd_ptcp_if_t; /* we just *implement* this interface */ /* the nsd_ptcp object */ struct nsd_ptcp_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - uchar *pRemHostIP; /**< IP address of remote peer (currently used in server mode, only) */ + prop_t *remoteIP; /**< IP address of remote peer (currently used in server mode, only) */ uchar *pRemHostName; /**< host name of remote peer (currently used in server mode, only) */ struct sockaddr_storage remAddr; /**< remote addr as sockaddr - used for legacy ACL code */ int sock; /**< the socket we use for regular, single-socket, operations */ diff --git a/runtime/obj-types.h b/runtime/obj-types.h index da27a391..30a6a2c0 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -282,14 +282,12 @@ rsRetVal objName##ClassExit(void) \ rsRetVal OBJ##Destruct(OBJ##_t __attribute__((unused)) **ppThis) \ { \ DEFiRet; \ - int iCancelStateSave; \ OBJ##_t *pThis; #define CODESTARTobjDestruct(OBJ) \ ASSERT(ppThis != NULL); \ pThis = *ppThis; \ - ISOBJ_TYPE_assert(pThis, OBJ); \ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + ISOBJ_TYPE_assert(pThis, OBJ); /* note: there was a long-time bug in the macro below that lead to *ppThis = NULL * only when the object was actually destructed. I discovered this issue during @@ -309,7 +307,6 @@ rsRetVal objName##ClassExit(void) \ free(pThis); \ } \ *ppThis = NULL; \ - pthread_setcancelstate(iCancelStateSave, NULL); \ RETiRet; \ } diff --git a/runtime/obj.c b/runtime/obj.c index eb151b67..63f1f38c 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -604,7 +604,7 @@ finalize_it: /* Deserialize a single property. Pointer must be positioned at begin of line. Whole line * up until the \n is read. */ -static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) +rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) { DEFiRet; number_t i; @@ -665,7 +665,7 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) if(c != '\n') ABORT_FINALIZE(RS_RET_INVALID_PROPFRAME); finalize_it: - if(Debug && iRet != RS_RET_OK) { + if(Debug && iRet != RS_RET_OK && iRet != RS_RET_NO_PROPLINE) { strm.GetCurrOffset(pStrm, &offs); dbgprintf("error %d deserializing property name, offset %lld, step %d\n", iRet, offs, step); @@ -767,21 +767,20 @@ finalize_it: * of the trailer. Header must already have been processed. * rgerhards, 2008-01-11 */ -static rsRetVal objDeserializeProperties(obj_t *pObj, objInfo_t *pObjInfo, strm_t *pStrm) +static rsRetVal objDeserializeProperties(obj_t *pObj, rsRetVal (*objSetProperty)(), strm_t *pStrm) { DEFiRet; var_t *pVar = NULL; ISOBJ_assert(pObj); ISOBJ_TYPE_assert(pStrm, strm); - ASSERT(pObjInfo != NULL); CHKiRet(var.Construct(&pVar)); CHKiRet(var.ConstructFinalize(pVar)); iRet = objDeserializeProperty(pVar, pStrm); while(iRet == RS_RET_OK) { - CHKiRet(pObjInfo->objMethods[objMethod_SETPROPERTY](pObj, pVar)); + CHKiRet(objSetProperty(pObj, pVar)); /* re-init var object - TODO: method of var! */ rsCStrDestruct(&pVar->pcsName); /* no longer needed */ if(pVar->varType == VARTYPE_STR) { @@ -848,7 +847,7 @@ Deserialize(void *ppObj, uchar *pszTypeExpected, strm_t *pStrm, rsRetVal (*fFixu CHKiRet(pObjInfo->objMethods[objMethod_CONSTRUCT](&pObj)); /* we got the object, now we need to fill the properties */ - CHKiRet(objDeserializeProperties(pObj, pObjInfo, pStrm)); + CHKiRet(objDeserializeProperties(pObj, pObjInfo->objMethods[objMethod_SETPROPERTY], pStrm)); /* check if we need to call a fixup function that modifies the object * before it is finalized. -- rgerhards, 2008-01-13 @@ -873,6 +872,104 @@ finalize_it: } +/* De-Serialize an object, with known constructur and destructor. Params like Deserialize(). + * rgerhards, 2012-11-03 + */ +rsRetVal +objDeserializeWithMethods(void *ppObj, uchar *pszTypeExpected, int lenTypeExpected, strm_t *pStrm, rsRetVal (*fFixup)(obj_t*,void*), void *pUsr, rsRetVal (*objConstruct)(), rsRetVal (*objConstructFinalize)(), rsRetVal (*objDeserialize)()) +{ + DEFiRet; + rsRetVal iRetLocal; + obj_t *pObj = NULL; + int oVers = 0; /* keep compiler happy, but it is totally useless but takes up some execution time... */ + cstr_t *pstrID = NULL; + + assert(ppObj != NULL); + assert(pszTypeExpected != NULL); + ISOBJ_TYPE_assert(pStrm, strm); + + /* we de-serialize the header. if all goes well, we are happy. However, if + * we experience a problem, we try to recover. We do this by skipping to + * the next object header. This is defined via the line-start cookies. In + * worst case, we exhaust the queue, but then we receive EOF return state, + * from objDeserializeTryRecover(), what will cause us to ultimately give up. + * rgerhards, 2008-07-08 + */ + do { + iRetLocal = objDeserializeHeader((uchar*) "Obj", &pstrID, &oVers, pStrm); + if(iRetLocal != RS_RET_OK) { + dbgprintf("objDeserialize error %d during header processing - " + "trying to recover\n", iRetLocal); + CHKiRet(objDeserializeTryRecover(pStrm)); + } + } while(iRetLocal != RS_RET_OK); + + if(rsCStrSzStrCmp(pstrID, pszTypeExpected, lenTypeExpected)) + ABORT_FINALIZE(RS_RET_INVALID_OID); + + CHKiRet(objConstruct(&pObj)); + + /* we got the object, now we need to fill the properties */ + CHKiRet(objDeserialize(pObj, pStrm)); + CHKiRet(objDeserializeTrailer(pStrm)); /* do trailer checks */ + + /* check if we need to call a fixup function that modifies the object + * before it is finalized. -- rgerhards, 2008-01-13 + */ + if(fFixup != NULL) + CHKiRet(fFixup(pObj, pUsr)); + + /* we have a valid object, let's finalize our work and return */ + if(objConstructFinalize != NULL) { + CHKiRet(objConstructFinalize(pObj)); + } + + *((obj_t**) ppObj) = pObj; + +finalize_it: + if(iRet != RS_RET_OK && pObj != NULL) + free(pObj); /* TODO: check if we can call destructor 2008-01-13 rger */ + + if(pstrID != NULL) + rsCStrDestruct(&pstrID); + + RETiRet; +} + +/* This is a dummy deserializer, to be used for the delete queue reader + * specifically. This is kind of a hack, but also to be replace (hopefully) soon + * by totally different code. So let's make it as simple as possible... + * rgerhards, 2012-11-06 + */ +rsRetVal +objDeserializeDummy(obj_t __attribute__((unused)) *pObj, strm_t *pStrm) +{ + DEFiRet; + var_t *pVar = NULL; + + CHKiRet(var.Construct(&pVar)); + CHKiRet(var.ConstructFinalize(pVar)); + + iRet = objDeserializeProperty(pVar, pStrm); + while(iRet == RS_RET_OK) { + /* this loop does actually NOGHTING but read the file... */ + /* re-init var object - TODO: method of var! */ + rsCStrDestruct(&pVar->pcsName); /* no longer needed */ + if(pVar->varType == VARTYPE_STR) { + if(pVar->val.pStr != NULL) + rsCStrDestruct(&pVar->val.pStr); + } + iRet = objDeserializeProperty(pVar, pStrm); + } +finalize_it: + if(iRet == RS_RET_NO_PROPLINE) + iRet = RS_RET_OK; /* NO_PROPLINE is OK and a kind of EOF! */ + if(pVar != NULL) + var.Destruct(&pVar); + RETiRet; +} + + /* De-Serialize an object, but treat it as property bag. * rgerhards, 2008-01-11 */ @@ -909,7 +1006,7 @@ objDeserializeObjAsPropBag(obj_t *pObj, strm_t *pStrm) CHKiRet(FindObjInfo(pstrID, &pObjInfo)); /* we got the object, now we need to fill the properties */ - CHKiRet(objDeserializeProperties(pObj, pObjInfo, pStrm)); + CHKiRet(objDeserializeProperties(pObj, pObjInfo->objMethods[objMethod_SETPROPERTY], pStrm)); finalize_it: if(pstrID != NULL) @@ -961,7 +1058,7 @@ DeserializePropBag(obj_t *pObj, strm_t *pStrm) CHKiRet(FindObjInfo(pstrID, &pObjInfo)); /* we got the object, now we need to fill the properties */ - CHKiRet(objDeserializeProperties(pObj, pObjInfo, pStrm)); + CHKiRet(objDeserializeProperties(pObj, pObjInfo->objMethods[objMethod_SETPROPERTY], pStrm)); finalize_it: if(pstrID != NULL) diff --git a/runtime/obj.h b/runtime/obj.h index 32f7ef09..27d32b7a 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -83,10 +83,7 @@ ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ; \ ((obj_t*) (pThis))->pszName = NULL #endif -#define objDestruct(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_DESTRUCT])(&pThis) #define objSerialize(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_SERIALIZE]) -#define objGetSeverity(pThis, piSever) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_GETSEVERITY])(pThis, piSever) -#define objDebugPrint(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_DEBUGPRINT])(pThis) #define OBJSetMethodHandler(methodID, pHdlr) \ CHKiRet(obj.InfoSetMethod(pObjInfoOBJ, methodID, (rsRetVal (*)(void*)) pHdlr)) @@ -121,6 +118,9 @@ ENDinterface(obj) rsRetVal objGetObjInterface(obj_if_t *pIf); PROTOTYPEObjClassInit(obj); PROTOTYPEObjClassExit(obj); +rsRetVal objDeserializeWithMethods(void *ppObj, uchar *pszTypeExpected, int lenTypeExpected, strm_t *pStrm, rsRetVal (*fFixup)(obj_t*,void*), void *pUsr, rsRetVal (*objConstruct)(), rsRetVal (*objConstructFinalize)(), rsRetVal (*objDeserialize)()); +rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm); +rsRetVal objDeserializeDummy(obj_t *pObj, strm_t *pStrm); /* the following definition is only for "friends" */ diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 9cf3781b..e63eb681 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -42,9 +42,7 @@ rsRetVal OMSRdestruct(omodStringRequest_t *pThis) /* free the strings */ if(pThis->ppTplName != NULL) { for(i = 0 ; i < pThis->iNumEntries ; ++i) { - if(pThis->ppTplName[i] != NULL) { - free(pThis->ppTplName[i]); - } + free(pThis->ppTplName[i]); } free(pThis->ppTplName); } diff --git a/runtime/parser.c b/runtime/parser.c index 645ea0f4..74b28f4c 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -143,6 +143,14 @@ finalize_it: RETiRet; } +void +printParserList(parserList_t *pList) +{ + while(pList != NULL) { + dbgprintf("parser: %s\n", pList->pParser->pName); + pList = pList->pNext; + } +} /* find a parser based on the provided name */ static rsRetVal @@ -354,11 +362,10 @@ SanitizeMsg(msg_t *pMsg) */ int bNeedSanitize = 0; for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { - if(iscntrl(pszMsg[iSrc])) { + if(pszMsg[iSrc] < 32) { if(bSpaceLFOnRcv && pszMsg[iSrc] == '\n') pszMsg[iSrc] = ' '; - else - if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + else if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { bNeedSanitize = 1; if (!bSpaceLFOnRcv) break; @@ -375,7 +382,9 @@ SanitizeMsg(msg_t *pMsg) FINALIZE; } - /* now copy over the message and sanitize it */ + /* now copy over the message and sanitize it. Note that up to iSrc-1 there was + * obviously no need to sanitize, so we can go over that quickly... + */ iMaxLine = glbl.GetMaxLine(); maxDest = lenMsg * 4; /* message can grow at most four-fold */ if(maxDest > iMaxLine) @@ -384,9 +393,13 @@ SanitizeMsg(msg_t *pMsg) pDst = szSanBuf; else CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); - iSrc = iDst = 0; + if(iSrc > 0) { + iSrc--; /* go back to where everything is OK */ + memcpy(pDst, pszMsg, iSrc); /* fast copy known good */ + } + iDst = iSrc; while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ - if(iscntrl((int) pszMsg[iSrc]) && (pszMsg[iSrc] != '\t' || bEscapeTab)) { + if((pszMsg[iSrc] < 32) && (pszMsg[iSrc] != '\t' || bEscapeTab)) { /* note: \0 must always be escaped, the rest of the code currently * can not handle it! -- rgerhards, 2009-08-26 */ diff --git a/runtime/parser.h b/runtime/parser.h index f214ba0c..87a6269e 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -62,6 +62,7 @@ BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */ ENDinterface(parser) #define parserCURR_IF_VERSION 1 /* increment whenever you change the interface above! */ +void printParserList(parserList_t *pList); /* prototypes */ PROTOTYPEObj(parser); diff --git a/runtime/prop.c b/runtime/prop.c index 9d5927fd..cb89fac0 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -100,7 +100,7 @@ static int GetStringLen(prop_t *pThis) /* get string */ -static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) +rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) { BEGINfunc ISOBJ_TYPE_assert(pThis, prop); diff --git a/runtime/prop.h b/runtime/prop.h index 40a35f9b..c7564e6b 100644 --- a/runtime/prop.h +++ b/runtime/prop.h @@ -52,6 +52,13 @@ ENDinterface(prop) #define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* get classic c-style string */ +static inline uchar * +propGetSzStr(prop_t *pThis) +{ + return(pThis->len < CONF_PROP_BUFSIZE) ? pThis->szVal.sz : pThis->szVal.psz; +} + /* prototypes */ PROTOTYPEObj(prop); diff --git a/runtime/queue.c b/runtime/queue.c index 09ba7e67..abe2be06 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -59,7 +59,6 @@ #include "datetime.h" #include "unicode-helper.h" #include "statsobj.h" -#include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS # include <sched.h> @@ -74,7 +73,7 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(statsobj) /* forward-definitions */ -static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); +static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, msg_t *pMsg); static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates); static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); @@ -83,7 +82,7 @@ static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti); static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti); static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub); static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub); -static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr); +static rsRetVal qAddDirect(qqueue_t *pThis, msg_t *pMsg); static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis); static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis); static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis); @@ -127,11 +126,11 @@ static struct cnfparamblk pblk = }; /* debug aid */ -static void displayBatchState(batch_t *pBatch) +static inline void displayBatchState(batch_t *pBatch) { int i; for(i = 0 ; i < pBatch->nElem ; ++i) { - DBGPRINTF("displayBatchState %p[%d]: %d\n", pBatch, i, pBatch->pElem[i].state); + DBGPRINTF("displayBatchState %p[%d]: %d\n", pBatch, i, pBatch->eltState[i]); } } @@ -232,12 +231,19 @@ getQueueTypeName(queueType_t t) switch(t) { case QUEUETYPE_FIXED_ARRAY: r = "FixedArray"; + break; case QUEUETYPE_LINKEDLIST: r = "LinkedList"; + break; case QUEUETYPE_DISK: r = "Disk"; + break; case QUEUETYPE_DIRECT: r = "Direct"; + break; + default: + r = "invalid/unknown queue mode"; + break; } return r; } @@ -308,16 +314,16 @@ getLogicalQueueSize(qqueue_t *pThis) */ static inline void queueDrain(qqueue_t *pThis) { - void *pUsr; + msg_t *pMsg; ASSERT(pThis != NULL); BEGINfunc DBGOPRINT((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ while(ATOMIC_DEC_AND_FETCH(&pThis->iQueueSize, &pThis->mutQueueSize) > 0) { - pThis->qDeq(pThis, &pUsr); - if(pUsr != NULL) { - objDestruct(pUsr); + pThis->qDeq(pThis, &pMsg); + if(pMsg != NULL) { + msgDestruct(&pMsg); } pThis->qDel(pThis); } @@ -407,7 +413,7 @@ StartDA(qqueue_t *pThis) */ pThis->pqDA->pqParent = pThis; - CHKiRet(qqueueSetpUsr(pThis->pqDA, pThis->pUsr)); + CHKiRet(qqueueSetpAction(pThis->pqDA, pThis->pAction)); CHKiRet(qqueueSetsizeOnDiskMax(pThis->pqDA, pThis->sizeOnDiskMax)); CHKiRet(qqueueSetiDeqSlowdown(pThis->pqDA, pThis->iDeqSlowdown)); CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); @@ -542,7 +548,7 @@ static rsRetVal qDestructFixedArray(qqueue_t *pThis) } -static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) +static rsRetVal qAddFixedArray(qqueue_t *pThis, msg_t* in) { DEFiRet; @@ -556,7 +562,7 @@ static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) } -static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out) +static rsRetVal qDeqFixedArray(qqueue_t *pThis, msg_t **out) { DEFiRet; @@ -617,7 +623,7 @@ static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) RETiRet; } -static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) +static rsRetVal qAddLinkedList(qqueue_t *pThis, msg_t* pMsg) { qLinkedList_t *pEntry; DEFiRet; @@ -625,7 +631,7 @@ static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) CHKmalloc((pEntry = (qLinkedList_t*) MALLOC(sizeof(qLinkedList_t)))); pEntry->pNext = NULL; - pEntry->pUsr = pUsr; + pEntry->pMsg = pMsg; if(pThis->tVars.linklist.pDelRoot == NULL) { pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = pEntry; @@ -643,14 +649,13 @@ finalize_it: } -static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr) +static rsRetVal qDeqLinkedList(qqueue_t *pThis, msg_t **ppMsg) { qLinkedList_t *pEntry; DEFiRet; pEntry = pThis->tVars.linklist.pDeqRoot; - ISOBJ_TYPE_assert(pEntry->pUsr, msg); - *ppUsr = pEntry->pUsr; + *ppMsg = pEntry->pMsg; pThis->tVars.linklist.pDeqRoot = pEntry->pNext; RETiRet; @@ -740,18 +745,12 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) { DEFiRet; strm_t *psQIF = NULL; - uchar pszQIFNam[MAXFNAME]; - size_t lenQIFNam; struct stat stat_buf; ISOBJ_TYPE_assert(pThis, qqueue); - /* Construct file name */ - lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", - (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - /* check if the file exists */ - if(stat((char*) pszQIFNam, &stat_buf) == -1) { + if(stat((char*) pThis->pszQIFNam, &stat_buf) == -1) { if(errno == ENOENT) { DBGOPRINT((obj_t*) pThis, "clean startup, no .qi file found\n"); ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); @@ -766,7 +765,7 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) CHKiRet(strm.Construct(&psQIF)); CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_READ)); CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.SetFName(psQIF, pThis->pszQIFNam, pThis->lenQIFNam)); CHKiRet(strm.ConstructFinalize(psQIF)); /* first, we try to read the property bag for ourselfs */ @@ -778,9 +777,7 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - /* create a duplicate for the read "pointer". - */ - + /* create a duplicate for the read "pointer". */ CHKiRet(strm.Dup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq)); CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */ CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); @@ -799,7 +796,7 @@ finalize_it: strm.Destruct(&psQIF); if(iRet != RS_RET_OK) { - DBGOPRINT((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", + DBGOPRINT((obj_t*) pThis, "state %d reading .qi file - can not read persisted info (if any)\n", iRet); } @@ -880,7 +877,8 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) DEFiRet; ASSERT(pThis != NULL); - + + free(pThis->pszQIFNam); if(pThis->tVars.disk.pWrite != NULL) strm.Destruct(&pThis->tVars.disk.pWrite); if(pThis->tVars.disk.pReadDeq != NULL) @@ -891,7 +889,7 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) RETiRet; } -static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) +static rsRetVal qAddDisk(qqueue_t *pThis, msg_t* pMsg) { DEFiRet; number_t nWriteCount; @@ -899,7 +897,7 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) ASSERT(pThis != NULL); CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); - CHKiRet((objSerialize(pUsr))(pUsr, pThis->tVars.disk.pWrite)); + CHKiRet((objSerialize(pMsg))(pMsg, pThis->tVars.disk.pWrite)); CHKiRet(strm.Flush(pThis->tVars.disk.pWrite)); CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ @@ -909,7 +907,7 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) * the in-memory representation. The instance will be re-created upon * dequeue. -- rgerhards, 2008-07-09 */ - objDestruct(pUsr); + msgDestruct(&pMsg); DBGOPRINT((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets, EnqOnly:%d\n", nWriteCount, pThis->tVars.disk.sizeOnDisk, pThis->bEnqOnly); @@ -919,43 +917,11 @@ finalize_it: } -static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr) +static rsRetVal qDeqDisk(qqueue_t *pThis, msg_t **ppMsg) { DEFiRet; - iRet = obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL); - RETiRet; -} - - -static rsRetVal qDelDisk(qqueue_t *pThis) -{ - obj_t *pDummyObj; /* we need to deserialize it... */ - DEFiRet; - - int64 offsIn; - int64 offsOut; - - CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsIn)); - CHKiRet(obj.Deserialize(&pDummyObj, (uchar*) "msg", pThis->tVars.disk.pReadDel, NULL, NULL)); - objDestruct(pDummyObj); - CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsOut)); - - /* This time it is a bit tricky: we free disk space only upon file deletion. So we need - * to keep track of what we have read until we get an out-offset that is lower than the - * in-offset (which indicates file change). Then, we can subtract the whole thing from - * the on-disk size. -- rgerhards, 2008-01-30 - */ - if(offsIn < offsOut) { - pThis->tVars.disk.bytesRead += offsOut - offsIn; - } else { - pThis->tVars.disk.sizeOnDisk -= pThis->tVars.disk.bytesRead; - pThis->tVars.disk.bytesRead = offsOut; - DBGOPRINT((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk); - /* awake possibly waiting enq process */ - pthread_cond_signal(&pThis->notFull); /* we hold the mutex while we are in here! */ - } - -finalize_it: + iRet = objDeserializeWithMethods(ppMsg, (uchar*) "msg", 3, pThis->tVars.disk.pReadDeq, NULL, + NULL, msgConstructForDeserializer, NULL, MsgDeserialize); RETiRet; } @@ -972,10 +938,12 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) return RS_RET_OK; } -static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) +static rsRetVal qAddDirect(qqueue_t *pThis, msg_t* pMsg) { batch_t singleBatch; batch_obj_t batchObj; + batch_state_t batchState = BATCH_STATE_RDY; + sbool active = 1; int i; DEFiRet; @@ -992,17 +960,17 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) */ memset(&batchObj, 0, sizeof(batch_obj_t)); memset(&singleBatch, 0, sizeof(batch_t)); - batchObj.state = BATCH_STATE_RDY; - batchObj.pUsrp = (obj_t*) pUsr; - batchObj.bFilterOK = 1; + batchObj.pMsg = pMsg; singleBatch.nElem = 1; /* there always is only one in direct mode */ singleBatch.pElem = &batchObj; - iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate); + singleBatch.eltState = &batchState; + singleBatch.active = &active; + iRet = pThis->pConsumer(pThis->pAction, &singleBatch, &pThis->bShutdownImmediate); /* delete the batch string params: TODO: create its own "class" for this */ for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) { free(batchObj.staticActStrings[i]); } - objDestruct(pUsr); + msgDestruct(&pMsg); RETiRet; } @@ -1024,7 +992,7 @@ rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch) * We use our knowledge about the batch_t structure below, but without that, we * pay a too-large performance toll... -- rgerhards, 2009-04-22 */ - iRet = pThis->pConsumer(pThis->pUsr, pBatch, &pThis->bShutdownImmediate); + iRet = pThis->pConsumer(pThis->pAction, pBatch, NULL); RETiRet; } @@ -1045,17 +1013,17 @@ static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) * things truely different. -- rgerhards, 2008-02-12 */ static rsRetVal -qqueueAdd(qqueue_t *pThis, void *pUsr) +qqueueAdd(qqueue_t *pThis, msg_t *pMsg) { DEFiRet; ASSERT(pThis != NULL); - CHKiRet(pThis->qAdd(pThis, pUsr)); + CHKiRet(pThis->qAdd(pThis, pMsg)); if(pThis->qType != QUEUETYPE_DIRECT) { ATOMIC_INC(&pThis->iQueueSize, &pThis->mutQueueSize); - DBGOPRINT((obj_t*) pThis, "entry added, size now log %d, phys %d entries\n", + DBGOPRINT((obj_t*) pThis, "qqueueAdd: entry added, size now log %d, phys %d entries\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); } @@ -1067,7 +1035,7 @@ finalize_it: /* generic code to dequeue a queue entry */ static rsRetVal -qqueueDeq(qqueue_t *pThis, void **ppUsr) +qqueueDeq(qqueue_t *pThis, msg_t **ppMsg) { DEFiRet; @@ -1078,7 +1046,7 @@ qqueueDeq(qqueue_t *pThis, void **ppUsr) * If we decrement, however, we may lose a message. But that is better than * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ - iRet = pThis->qDeq(pThis, ppUsr); + iRet = pThis->qDeq(pThis, ppMsg); ATOMIC_INC(&pThis->nLogDeq, &pThis->mutLogDeq); // DBGOPRINT((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", @@ -1176,11 +1144,11 @@ tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) rsRetVal iRetLocal; DEFiRet; -RUNLOG_STR("trying to shutdown workers within Action Timeout"); ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ /* instruct workers to finish ASAP, even if still work exists */ + DBGOPRINT((obj_t*) pThis, "trying to shutdown workers within Action Timeout"); DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode\n"); pThis->bEnqOnly = 1; pThis->bShutdownImmediate = 1; @@ -1324,8 +1292,6 @@ finalize_it: RETiRet; } - - /* Constructor for the queue object * This constructs the data structure, but does not yet start the queue. That * is done by queueStart(). The reason is that we want to give the caller a chance @@ -1374,7 +1340,7 @@ finalize_it: } -/* set default inisde queue object suitable for action queues. +/* set default inside queue object suitable for action queues. * This shall be called directly after queue construction. This functions has * been added in support of the new v6 config system. It expect properly pre-initialized * objects, but we need to differentiate between ruleset main and action queues. @@ -1408,6 +1374,36 @@ qqueueSetDefaultsActionQueue(qqueue_t *pThis) } +/* set defaults inside queue object suitable for main/ruleset queues. + * See queueSetDefaultsActionQueue() for more details and background. + */ +void +qqueueSetDefaultsRulesetQueue(qqueue_t *pThis) +{ + pThis->qType = QUEUETYPE_FIXED_ARRAY; /* type of the main message queue above */ + pThis->iMaxQueueSize = 50000; /* size of the main message queue above */ + pThis->iDeqBatchSize = 1024; /* default batch size */ + pThis->iHighWtrMrk = 45000; /* high water mark for disk-assisted queues */ + pThis->iLowWtrMrk = 20000; /* low water mark for disk-assisted queues */ + pThis->iDiscardMrk = 49500; /* begin to discard messages */ + pThis->iDiscardSeverity = 8; /* turn off */ + pThis->iNumWorkerThreads = 1; /* number of worker threads for the mm queue above */ + pThis->iMaxFileSize = 16*1024*1024; + pThis->iPersistUpdCnt = 0; /* persist queue info every n updates */ + pThis->bSyncQueueFiles = 0; + pThis->toQShutdown = 1500; /* queue shutdown */ + pThis->toActShutdown = 1000; /* action shutdown (in phase 2) */ + pThis->toEnq = 2000; /* timeout for queue enque */ + pThis->toWrkShutdown = 60000; /* timeout for worker thread shutdown */ + pThis->iMinMsgsPerWrkr = 1000; /* minimum messages per worker needed to start a new one */ + pThis->bSaveOnShutdown = 1; /* save queue on shutdown (when DA enabled)? */ + pThis->sizeOnDiskMax = 0; /* unlimited */ + pThis->iDeqSlowdown = 0; + pThis->iDeqtWinFromHr = 0; + pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */ +} + + /* This function checks if the provided message shall be discarded and does so, if needed. * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to * provide real-time creation of spool files. @@ -1421,22 +1417,21 @@ qqueueSetDefaultsActionQueue(qqueue_t *pThis) * the return state! * rgerhards, 2008-01-24 */ -static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, void *pUsr) +static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, msg_t *pMsg) { DEFiRet; rsRetVal iRetLocal; int iSeverity; ISOBJ_TYPE_assert(pThis, qqueue); - ISOBJ_assert(pUsr); if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk) { - iRetLocal = objGetSeverity(pUsr, &iSeverity); + iRetLocal = MsgGetSeverity(pMsg, &iSeverity); if(iRetLocal == RS_RET_OK && iSeverity >= pThis->iDiscardSeverity) { DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", iQueueSize, iSeverity); STATSCOUNTER_INC(pThis->ctrNFDscrd, pThis->mutCtrNFDscrd); - objDestruct(pUsr); + msgDestruct(&pMsg); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } else { DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg " @@ -1455,19 +1450,38 @@ static inline rsRetVal DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) { int i; + off64_t bytesDel; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); /* now send delete request to storage driver */ - for(i = 0 ; i < nElem ; ++i) { - pThis->qDel(pThis); + if(pThis->qType == QUEUETYPE_DISK) { + strmMultiFileSeek(pThis->tVars.disk.pReadDel, pThis->tVars.disk.deqFileNumOut, + pThis->tVars.disk.deqOffs, &bytesDel); + /* We need to correct the on-disk file size. This time it is a bit tricky: + * we free disk space only upon file deletion. So we need to keep track of what we + * have read until we get an out-offset that is lower than the in-offset (which + * indicates file change). Then, we can subtract the whole thing from the on-disk + * size. -- rgerhards, 2008-01-30 + */ + if(bytesDel != 0) { + pThis->tVars.disk.sizeOnDisk -= bytesDel; + DBGOPRINT((obj_t*) pThis, "doDeleteBatch: a %lld octet file has been deleted, now %lld octets disk " + "space used\n", bytesDel, pThis->tVars.disk.sizeOnDisk); + /* awake possibly waiting enq process */ + pthread_cond_signal(&pThis->notFull); /* we hold the mutex while we are in here! */ + } + } else { /* memory queue */ + for(i = 0 ; i < nElem ; ++i) { + pThis->qDel(pThis); + } } /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ ATOMIC_SUB(&pThis->iQueueSize, nElem, &pThis->mutQueueSize); ATOMIC_SUB(&pThis->nLogDeq, nElem, &pThis->mutLogDeq); - DBGPRINTF("delete batch from store, new sizes: log %d, phys %d\n", + DBGPRINTF("doDeleteBatch: delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); ++pThis->deqIDDel; /* one more batch dequeued */ @@ -1522,7 +1536,7 @@ static inline rsRetVal DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) { int i; - void *pUsr; + msg_t *pMsg; int nEnqueued = 0; rsRetVal localRet; DEFiRet; @@ -1531,20 +1545,19 @@ DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) assert(pBatch != NULL); for(i = 0 ; i < pBatch->nElem ; ++i) { - pUsr = pBatch->pElem[i].pUsrp; - if( pBatch->pElem[i].state == BATCH_STATE_RDY - || pBatch->pElem[i].state == BATCH_STATE_SUB) { - localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY, - (obj_t*)MsgAddRef((msg_t*) pUsr)); + pMsg = pBatch->pElem[i].pMsg; + if( pBatch->eltState[i] == BATCH_STATE_RDY + || pBatch->eltState[i] == BATCH_STATE_SUB) { + localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY, MsgAddRef(pMsg)); ++nEnqueued; if(localRet != RS_RET_OK) { - DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet); + DBGPRINTF("DeleteProcessedBatch: error %d re-enqueuing unprocessed data element - discarded\n", localRet); } } - objDestruct(pUsr); + msgDestruct(&pMsg); } - DBGPRINTF("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued); + DBGPRINTF("DeleteProcessedBatch: we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued); if(nEnqueued > 0) qqueueChkPersist(pThis, nEnqueued); @@ -1573,7 +1586,7 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz int nDiscarded; int nDeleted; int iQueueSize; - void *pUsr; + msg_t *pMsg; rsRetVal localRet; DEFiRet; @@ -1581,11 +1594,14 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz DeleteProcessedBatch(pThis, &pWti->batch); nDequeued = nDiscarded = 0; + if(pThis->qType == QUEUETYPE_DISK) { + pThis->tVars.disk.deqFileNumIn = strmGetCurrFileNum(pThis->tVars.disk.pReadDeq); + } while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { - CHKiRet(qqueueDeq(pThis, &pUsr)); + CHKiRet(qqueueDeq(pThis, &pMsg)); /* check if we should discard this element */ - localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr); + localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pMsg); if(localRet == RS_RET_QUEUE_FULL) { ++nDiscarded; continue; @@ -1594,12 +1610,16 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz } /* all well, use this element */ - pWti->batch.pElem[nDequeued].pUsrp = pUsr; - pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY; - pWti->batch.pElem[nDequeued].bFilterOK = 1; // TODO: think again if we can handle that with more performance + pWti->batch.pElem[nDequeued].pMsg = pMsg; + pWti->batch.eltState[nDequeued] = BATCH_STATE_RDY; ++nDequeued; } + if(pThis->qType == QUEUETYPE_DISK) { + strm.GetCurrOffset(pThis->tVars.disk.pReadDeq, &pThis->tVars.disk.deqOffs); + pThis->tVars.disk.deqFileNumOut = strmGetCurrFileNum(pThis->tVars.disk.pReadDeq); + } + /* it is sufficient to persist only when the bulk of work is done */ qqueueChkPersist(pThis, nDequeued+nDiscarded+nDeleted); @@ -1607,7 +1627,6 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz pWti->batch.nElemDeq = nDequeued + nDiscarded; pWti->batch.deqID = getNextDeqID(pThis); *piRemainingQueueSize = iQueueSize; - finalize_it: RETiRet; } @@ -1643,7 +1662,6 @@ DequeueConsumable(qqueue_t *pThis, wti_t *pWti) pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk); } - // TODO: MULTI: check physical queue size? pthread_cond_signal(&pThis->notFull); /* WE ARE NO LONGER PROTECTED BY THE MUTEX */ @@ -1837,7 +1855,8 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti) /* at this spot, we may be cancelled */ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); - CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch, &pThis->bShutdownImmediate)); + + CHKiRet(pThis->pConsumer(pThis->pAction, &pWti->batch, &pThis->bShutdownImmediate)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 @@ -1895,23 +1914,55 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti) /* iterate over returned results and enqueue them in DA queue */ for(i = 0 ; i < pWti->batch.nElem && !pThis->bShutdownImmediate ; i++) { - /* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs - * the message. So far, we simply assume we always have msg_t, what currently is always the case. - * rgerhards, 2009-05-28 - */ - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, - (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp)))); - pWti->batch.pElem[i].state = BATCH_STATE_COMM; /* commited to other queue! */ + iRet = qqueueEnqMsg(pThis->pqDA, eFLOWCTL_NO_DELAY, MsgAddRef(pWti->batch.pElem[i].pMsg)); + if(iRet != RS_RET_OK) { + if(iRet == RS_RET_ERR_QUEUE_EMERGENCY) { + /* Queue emergency error occured */ + DBGOPRINT((obj_t*) pThis, "ConsumerDA:qqueueEnqMsg caught RS_RET_ERR_QUEUE_EMERGENCY, aborting loop.\n"); + FINALIZE; + } else { + DBGOPRINT((obj_t*) pThis, "ConsumerDA:qqueueEnqMsg item (%d) returned with error state: '%d'\n", i, iRet); + } + } + pWti->batch.eltState[i] = BATCH_STATE_COMM; /* commited to other queue! */ } /* but now cancellation is no longer permitted */ pthread_setcancelstate(iCancelStateSave, NULL); finalize_it: + /* Check the last return state of qqueueEnqMsg. If an error was returned, we acknowledge it only. + * Unless the error code is RS_RET_ERR_QUEUE_EMERGENCY, we reset the return state to RS_RET_OK. + * Otherwise the Caller functions would run into an infinite Loop trying to enqueue the + * same messages over and over again. + * + * However we do NOT overwrite positive return states like + * RS_RET_TERMINATE_NOW, + * RS_RET_NO_RUN, + * RS_RET_IDLE, + * RS_RET_TERMINATE_WHEN_IDLE + * These return states are important for Queue handling of the upper laying functions. + * RGer: Note that checking for iRet < 0 is a bit bold. In theory, positive iRet + * values are "OK" states, and things that the caller shall deal with. However, + * this has not been done so consistently. Andre convinced me that the current + * code is an elegant solution. However, if problems with queue workers and/or + * shutdown come up, this code here should be looked at suspiciously. In those + * cases it may work out to check all status codes explicitely, just to avoid + * a pitfall due to unexpected states being passed on to the caller. + */ + if( iRet != RS_RET_OK && + iRet != RS_RET_ERR_QUEUE_EMERGENCY && + iRet < 0) { + DBGOPRINT((obj_t*) pThis, "ConsumerDA:qqueueEnqMsg Resetting iRet from %d back to RS_RET_OK\n", iRet); + iRet = RS_RET_OK; + } else { + DBGOPRINT((obj_t*) pThis, "ConsumerDA:qqueueEnqMsg returns with iRet %d\n", iRet); + } + /* now we are done, but potentially need to re-aquire the mutex */ if(bNeedReLock) d_pthread_mutex_lock(pThis->mut); - DBGOPRINT((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); + RETiRet; } @@ -1981,6 +2032,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; uchar pszBuf[64]; + uchar pszQIFNam[MAXFNAME]; int wrk; uchar *qName; size_t lenBuf; @@ -2003,8 +2055,8 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->qConstruct = qConstructLinkedList; pThis->qDestruct = qDestructLinkedList; pThis->qAdd = qAddLinkedList; - pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList; - pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList; + pThis->qDeq = qDeqLinkedList; + pThis->qDel = qDelLinkedList; pThis->MultiEnq = qqueueMultiEnqObjNonDirect; break; case QUEUETYPE_DISK: @@ -2012,10 +2064,16 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->qDestruct = qDestructDisk; pThis->qAdd = qAddDisk; pThis->qDeq = qDeqDisk; - pThis->qDel = qDelDisk; + pThis->qDel = NULL; /* delete for disk handled via special code! */ pThis->MultiEnq = qqueueMultiEnqObjNonDirect; /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ + /* pre-construct file name for .qi file */ + pThis->lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), + "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); + pThis->pszQIFNam = ustrdup(pszQIFNam); + DBGOPRINT((obj_t*) pThis, ".qi file name is '%s', len %d\n", pThis->pszQIFNam, + (int) pThis->lenQIFNam); break; case QUEUETYPE_DIRECT: pThis->qConstruct = qConstructDirect; @@ -2075,6 +2133,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, pThis->iDeqBatchSize); + pThis->bQueueStarted = 1; if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ @@ -2105,7 +2164,6 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ * the case when a disk queue has been loaded. If we did not start it here, it would never start. */ qqueueAdviseMaxWorkers(pThis); - pThis->bQueueStarted = 1; /* support statistics gathering */ qName = obj.GetName((obj_t*)pThis); @@ -2142,7 +2200,7 @@ finalize_it: } -/* persist the queue to disk. If we have something to persist, we first +/* persist the queue to disk (write the .qi file). If we have something to persist, we first * save the information on the queue properties itself and then we call * the queue-type specific drivers. * Variable bIsCheckpoint is set to 1 if the persist is for a checkpoint, @@ -2153,8 +2211,6 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) { DEFiRet; strm_t *psQIF = NULL; /* Queue Info File */ - uchar pszQIFNam[MAXFNAME]; - size_t lenQIFNam; ASSERT(pThis != NULL); @@ -2172,13 +2228,9 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) DBGOPRINT((obj_t*) pThis, "persisting queue to disk, %d entries...\n", getPhysicalQueueSize(pThis)); - /* Construct file name */ - lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", - (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - if((bIsCheckpoint != QUEUE_CHECKPOINT) && (getPhysicalQueueSize(pThis) == 0)) { if(pThis->bNeedDelQIF) { - unlink((char*)pszQIFNam); + unlink((char*)pThis->pszQIFNam); pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ @@ -2191,7 +2243,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetbSync(psQIF, pThis->bSyncQueueFiles)); CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.SetFName(psQIF, pThis->pszQIFNam, pThis->lenQIFNam)); CHKiRet(strm.ConstructFinalize(psQIF)); /* first, write the property bag for ourselfs @@ -2203,7 +2255,6 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) CHKiRet(obj.BeginSerializePropBag(psQIF, (obj_t*) pThis)); objSerializeSCALAR(psQIF, iQueueSize, INT); objSerializeSCALAR(psQIF, tVars.disk.sizeOnDisk, INT64); - objSerializeSCALAR(psQIF, tVars.disk.bytesRead, INT64); CHKiRet(obj.EndSerialize(psQIF)); /* now persist the stream info */ @@ -2303,73 +2354,75 @@ DoSaveOnShutdown(qqueue_t *pThis) /* destructor for the queue object */ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) - /* shut down all workers - * We do not need to shutdown workers when we are in enqueue-only mode or we are a - * direct queue - because in both cases we have none... ;) - * with a child! -- rgerhards, 2008-01-28 - */ - if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL - && pThis->pWtpReg != NULL) - ShutdownWorkers(pThis); + if(pThis->bQueueStarted) { + /* shut down all workers + * We do not need to shutdown workers when we are in enqueue-only mode or we are a + * direct queue - because in both cases we have none... ;) + * with a child! -- rgerhards, 2008-01-28 + */ + if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL + && pThis->pWtpReg != NULL) + ShutdownWorkers(pThis); - if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { - CHKiRet(DoSaveOnShutdown(pThis)); - } + if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + CHKiRet(DoSaveOnShutdown(pThis)); + } - /* finally destruct our (regular) worker thread pool - * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, - * e.g. when they are not created in enqueue-only mode. We already check the condition - * as this may otherwise be very hard to find once we optimize (and have long forgotten - * about this condition here ;) - * rgerhards, 2008-01-25 - */ - if(pThis->qType != QUEUETYPE_DIRECT && pThis->pWtpReg != NULL) { - wtpDestruct(&pThis->pWtpReg); - } + /* finally destruct our (regular) worker thread pool + * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, + * e.g. when they are not created in enqueue-only mode. We already check the condition + * as this may otherwise be very hard to find once we optimize (and have long forgotten + * about this condition here ;) + * rgerhards, 2008-01-25 + */ + if(pThis->qType != QUEUETYPE_DIRECT && pThis->pWtpReg != NULL) { + wtpDestruct(&pThis->pWtpReg); + } - /* Now check if we actually have a DA queue and, if so, destruct it. - * Note that the wtp must be destructed first, it may be in cancel cleanup handler - * *right now* and actually *need* to access the queue object to persist some final - * data (re-queueing case). So we need to destruct the wtp first, which will make - * sure all workers have terminated. Please note that this also generates a situation - * where it is possible that the DA queue has a parent pointer but the parent has - * no WtpDA associated with it - which is perfectly legal thanks to this code here. - */ - if(pThis->pWtpDA != NULL) { - wtpDestruct(&pThis->pWtpDA); - } - if(pThis->pqDA != NULL) { - qqueueDestruct(&pThis->pqDA); - } + /* Now check if we actually have a DA queue and, if so, destruct it. + * Note that the wtp must be destructed first, it may be in cancel cleanup handler + * *right now* and actually *need* to access the queue object to persist some final + * data (re-queueing case). So we need to destruct the wtp first, which will make + * sure all workers have terminated. Please note that this also generates a situation + * where it is possible that the DA queue has a parent pointer but the parent has + * no WtpDA associated with it - which is perfectly legal thanks to this code here. + */ + if(pThis->pWtpDA != NULL) { + wtpDestruct(&pThis->pWtpDA); + } + if(pThis->pqDA != NULL) { + qqueueDestruct(&pThis->pqDA); + } - /* persist the queue (we always do that - queuePersits() does cleanup if the queue is empty) - * This handler is most important for disk queues, it will finally persist the necessary - * on-disk structures. In theory, other queueing modes may implement their other (non-DA) - * methods of persisting a queue between runs, but in practice all of this is done via - * disk queues and DA mode. Anyhow, it doesn't hurt to know that we could extend it here - * if need arises (what I doubt...) -- rgerhards, 2008-01-25 - */ - CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { - DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); - } + /* persist the queue (we always do that - queuePersits() does cleanup if the queue is empty) + * This handler is most important for disk queues, it will finally persist the necessary + * on-disk structures. In theory, other queueing modes may implement their other (non-DA) + * methods of persisting a queue between runs, but in practice all of this is done via + * disk queues and DA mode. Anyhow, it doesn't hurt to know that we could extend it here + * if need arises (what I doubt...) -- rgerhards, 2008-01-25 + */ + CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { + DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); + } - /* finally, clean up some simple things... */ - if(pThis->pqParent == NULL) { - /* if we are not a child, we allocated our own mutex, which we now need to destroy */ - pthread_mutex_destroy(pThis->mut); - free(pThis->mut); - } - pthread_mutex_destroy(&pThis->mutThrdMgmt); - pthread_cond_destroy(&pThis->notFull); - pthread_cond_destroy(&pThis->notEmpty); - pthread_cond_destroy(&pThis->belowFullDlyWtrMrk); - pthread_cond_destroy(&pThis->belowLightDlyWtrMrk); + /* finally, clean up some simple things... */ + if(pThis->pqParent == NULL) { + /* if we are not a child, we allocated our own mutex, which we now need to destroy */ + pthread_mutex_destroy(pThis->mut); + free(pThis->mut); + } + pthread_mutex_destroy(&pThis->mutThrdMgmt); + pthread_cond_destroy(&pThis->notFull); + pthread_cond_destroy(&pThis->notEmpty); + pthread_cond_destroy(&pThis->belowFullDlyWtrMrk); + pthread_cond_destroy(&pThis->belowLightDlyWtrMrk); - DESTROY_ATOMIC_HELPER_MUT(pThis->mutQueueSize); - DESTROY_ATOMIC_HELPER_MUT(pThis->mutLogDeq); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutQueueSize); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutLogDeq); - /* type-specific destructor */ - iRet = pThis->qDestruct(pThis); + /* type-specific destructor */ + iRet = pThis->qDestruct(pThis); + } free(pThis->pszFilePrefix); free(pThis->pszSpoolDir); @@ -2431,7 +2484,7 @@ finalize_it: * rgerhards, 2009-06-16 */ static inline rsRetVal -doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) +doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, msg_t *pMsg) { DEFiRet; int err; @@ -2440,7 +2493,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) STATSCOUNTER_INC(pThis->ctrEnqueued, pThis->mutCtrEnqueued); /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ - CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pMsg)); /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. @@ -2475,7 +2528,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) * In any case, this was the old code (if we do the TODO): * pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); */ - DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message " + DBGOPRINT((obj_t*) pThis, "doEnqSingleObject: FullDelay mark reached for full delayable message " "- blocking, queue size is %d.\n", pThis->iQueueSize); timeoutComp(&t, 1000); err = pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); @@ -2492,7 +2545,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) } } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY && !glbl.GetGlobalInputTermState()) { if(pThis->iQueueSize >= pThis->iLightDlyMrk) { - DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light " + DBGOPRINT((obj_t*) pThis, "doEnqSingleObject: LightDelay mark reached for light " "delayable message - blocking a bit.\n"); timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ err = pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); @@ -2515,29 +2568,31 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { STATSCOUNTER_INC(pThis->ctrFull, pThis->mutCtrFull); if(pThis->toEnq == 0 || pThis->bEnqOnly) { - DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - configured for immediate discarding.\n"); + DBGOPRINT((obj_t*) pThis, "doEnqSingleObject: queue FULL - configured for immediate discarding QueueSize=%d " + "MaxQueueSize=%d sizeOnDisk=%lld sizeOnDiskMax=%lld\n", pThis->iQueueSize, pThis->iMaxQueueSize, + pThis->tVars.disk.sizeOnDisk, pThis->sizeOnDiskMax); STATSCOUNTER_INC(pThis->ctrFDscrd, pThis->mutCtrFDscrd); - objDestruct(pUsr); + msgDestruct(&pMsg); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } else { - DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting %dms to drain.\n", pThis->toEnq); + DBGOPRINT((obj_t*) pThis, "doEnqSingleObject: queue FULL - waiting %dms to drain.\n", pThis->toEnq); if(glbl.GetGlobalInputTermState()) { - DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL, discard due to FORCE_TERM.\n"); + DBGOPRINT((obj_t*) pThis, "doEnqSingleObject: queue FULL, discard due to FORCE_TERM.\n"); ABORT_FINALIZE(RS_RET_FORCE_TERM); } timeoutComp(&t, pThis->toEnq); if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { - DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); + DBGOPRINT((obj_t*) pThis, "doEnqSingleObject: cond timeout, dropping message!\n"); STATSCOUNTER_INC(pThis->ctrFDscrd, pThis->mutCtrFDscrd); - objDestruct(pUsr); + msgDestruct(&pMsg); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } - dbgoprint((obj_t*) pThis, "enqueueMsg: wait solved queue full condition, enqueing\n"); + dbgoprint((obj_t*) pThis, "doEnqSingleObject: wait solved queue full condition, enqueing\n"); } } /* and finally enqueue the message */ - CHKiRet(qqueueAdd(pThis, pUsr)); + CHKiRet(qqueueAdd(pThis, pMsg)); STATSCOUNTER_SETMAX_NOMUT(pThis->ctrMaxqsize, pThis->iQueueSize); finalize_it: @@ -2613,11 +2668,11 @@ finalize_it: * Enqueues the new element and awakes worker thread. */ rsRetVal -qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr) +qqueueEnqMsgDirect(qqueue_t *pThis, msg_t *pMsg) { DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); - iRet = qAddDirect(pThis, pUsr); + iRet = qAddDirect(pThis, pMsg); RETiRet; } @@ -2626,7 +2681,7 @@ qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr) * Enqueues the new element and awakes worker thread. */ rsRetVal -qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) +qqueueEnqMsg(qqueue_t *pThis, flowControl_t flowCtlType, msg_t *pMsg) { DEFiRet; int iCancelStateSave; @@ -2638,7 +2693,7 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) d_pthread_mutex_lock(pThis->mut); } - CHKiRet(doEnqSingleObj(pThis, flowCtlType, pUsr)); + CHKiRet(doEnqSingleObj(pThis, flowCtlType, pMsg)); qqueueChkPersist(pThis, 1); @@ -2669,6 +2724,15 @@ qqueueDoCnfParams(struct nvlst *lst, struct cnfparamvals **ppvals) return RS_RET_OK; } + +/* are any queue params set at all? 1 - yes, 0 - no */ +int +queueCnfParamsSet(struct cnfparamvals *pvals) +{ + return cnfparamvalsIsSet(&pblk, pvals); +} + + /* apply all params from param block to queue. Must be called before * finalizing. This supports the v6 config system. Defaults were already * set during queue creation. The pvals object is destructed by this @@ -2735,6 +2799,12 @@ qqueueApplyCnfParam(qqueue_t *pThis, struct cnfparamvals *pvals) "param '%s'\n", pblk.descr[i].name); } } + if(pThis->qType == QUEUETYPE_DISK && pThis->pszFilePrefix == NULL) { + errmsg.LogError(0, RS_RET_QUEUE_DISK_NO_FN, "error on queue '%s', disk mode selected, but " + "no queue file name given; queue type changed to 'linkedList'", + obj.GetName((obj_t*) pThis)); + pThis->qType = QUEUETYPE_LINKEDLIST; + } cnfparamvalsDestruct(pvals, &pblk); return RS_RET_OK; } @@ -2758,7 +2828,7 @@ DEFpropSetMeth(qqueue, iLightDlyMrk, int) DEFpropSetMeth(qqueue, bIsDA, int) DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int) DEFpropSetMeth(qqueue, bSaveOnShutdown, int) -DEFpropSetMeth(qqueue, pUsr, void*) +DEFpropSetMeth(qqueue, pAction, action_t*) DEFpropSetMeth(qqueue, iDeqSlowdown, int) DEFpropSetMeth(qqueue, iDeqBatchSize, int) DEFpropSetMeth(qqueue, sizeOnDiskMax, int64) @@ -2781,8 +2851,6 @@ static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp) pThis->iQueueSize = pProp->val.num; } else if(isProp("tVars.disk.sizeOnDisk")) { pThis->tVars.disk.sizeOnDisk = pProp->val.num; - } else if(isProp("tVars.disk.bytesRead")) { - pThis->tVars.disk.bytesRead = pProp->val.num; } else if(isProp("qType")) { if(pThis->qType != pProp->val.num) ABORT_FINALIZE(RS_RET_QTYPE_MISMATCH); diff --git a/runtime/queue.h b/runtime/queue.h index edb770c6..886fac8d 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -51,7 +51,7 @@ typedef enum { /* list member definition for linked list types of queues: */ typedef struct qLinkedList_S { struct qLinkedList_S *pNext; - void *pUsr; + msg_t *pMsg; } qLinkedList_t; @@ -71,7 +71,7 @@ struct queue_s { int iMinMsgsPerWrkr;/* minimum nbr of msgs per worker thread, if more, a new worker is started until max wrkrs */ wtp_t *pWtpDA; wtp_t *pWtpReg; - void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ + action_t *pAction; /* for action queues, ptr to action object; for main queues unused */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ sbool bSyncQueueFiles;/* if working with files, sync them after each write? */ @@ -111,8 +111,8 @@ struct queue_s { /* type-specific handlers (set during construction) */ rsRetVal (*qConstruct)(struct queue_s *pThis); rsRetVal (*qDestruct)(struct queue_s *pThis); - rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr); - rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qAdd)(struct queue_s *pThis, msg_t *pMsg); + rsRetVal (*qDeq)(struct queue_s *pThis, msg_t **ppMsg); rsRetVal (*qDel)(struct queue_s *pThis); /* end type-specific handler */ /* public entry points (set during construction, permit to set best algorithm for params selected) */ @@ -135,6 +135,8 @@ struct queue_s { size_t lenSpoolDir; uchar *pszFilePrefix; size_t lenFilePrefix; + uchar *pszQIFNam; /* full .qi file name, based on parts above */ + size_t lenQIFNam; int iNumberFiles; /* how many files make up the queue? */ int64 iMaxFileSize; /* max size for a single queue file */ int64 sizeOnDiskMax; /* maximum size on disk allowed */ @@ -145,7 +147,8 @@ struct queue_s { struct queue_s *pqParent;/* pointer to the parent (if this is a child queue) */ int bDAEnqOnly; /* EnqOnly setting for DA queue */ /* now follow queueing mode specific data elements */ - union { /* different data elements based on queue type (qType) */ + //union { /* different data elements based on queue type (qType) */ + struct { /* different data elements based on queue type (qType) */ struct { long deqhead, head, tail; void** pBuf; /* the queued user data structure */ @@ -157,7 +160,9 @@ struct queue_s { } linklist; struct { int64 sizeOnDisk; /* current amount of disk space used */ - int64 bytesRead; /* number of bytes read from current (undeleted!) file */ + int64 deqOffs; /* offset after dequeue batch - used for file deleter */ + int deqFileNumIn; /* same for the circular file numbers, mainly for */ + int deqFileNumOut;/* deleting finished files */ strm_t *pWrite; /* current file to be written */ strm_t *pReadDeq; /* current file for dequeueing */ strm_t *pReadDel; /* current file for deleting */ @@ -184,8 +189,8 @@ struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); -rsRetVal qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr); -rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); +rsRetVal qqueueEnqMsgDirect(qqueue_t *pThis, msg_t *pMsg); +rsRetVal qqueueEnqMsg(qqueue_t *pThis, flowControl_t flwCtlType, msg_t *pMsg); rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); @@ -193,7 +198,9 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*)); rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch); rsRetVal qqueueDoCnfParams(struct nvlst *lst, struct cnfparamvals **ppvals); +int queueCnfParamsSet(struct cnfparamvals *pvals); rsRetVal qqueueApplyCnfParam(qqueue_t *pThis, struct cnfparamvals *pvals); +void qqueueSetDefaultsRulesetQueue(qqueue_t *pThis); void qqueueSetDefaultsActionQueue(qqueue_t *pThis); void qqueueDbgPrint(qqueue_t *pThis); @@ -213,7 +220,7 @@ PROTOTYPEpropSetMeth(qqueue, iDiscardMrk, int); PROTOTYPEpropSetMeth(qqueue, iDiscardSeverity, int); PROTOTYPEpropSetMeth(qqueue, iMinMsgsPerWrkr, int); PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int); -PROTOTYPEpropSetMeth(qqueue, pUsr, void*); +PROTOTYPEpropSetMeth(qqueue, pAction, action_t*); PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int); PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); PROTOTYPEpropSetMeth(qqueue, iDeqBatchSize, int); diff --git a/runtime/ratelimit.c b/runtime/ratelimit.c new file mode 100644 index 00000000..a808e04a --- /dev/null +++ b/runtime/ratelimit.c @@ -0,0 +1,385 @@ +/* ratelimit.c + * support for rate-limiting sources, including "last message + * repeated n times" processing. + * + * Copyright 2012 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include "rsyslog.h" +#include "errmsg.h" +#include "ratelimit.h" +#include "datetime.h" +#include "parser.h" +#include "unicode-helper.h" +#include "msg.h" +#include "rsconf.h" +#include "dirty.h" + +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(datetime) +DEFobjCurrIf(parser) + +/* static data */ + +/* generate a "repeated n times" message */ +static inline msg_t * +ratelimitGenRepMsg(ratelimit_t *ratelimit) +{ + msg_t *repMsg; + size_t lenRepMsg; + uchar szRepMsg[1024]; + + if(ratelimit->nsupp == 1) { /* we simply use the original message! */ + repMsg = MsgAddRef(ratelimit->pMsg); + } else {/* we need to duplicate, original message may still be in use in other + * parts of the system! */ + if((repMsg = MsgDup(ratelimit->pMsg)) == NULL) { + DBGPRINTF("Message duplication failed, dropping repeat message.\n"); + goto done; + } + lenRepMsg = snprintf((char*)szRepMsg, sizeof(szRepMsg), + " message repeated %d times: [%.800s]", + ratelimit->nsupp, getMSG(ratelimit->pMsg)); + MsgReplaceMSG(repMsg, szRepMsg, lenRepMsg); + } + +done: return repMsg; +} + +static inline rsRetVal +doLastMessageRepeatedNTimes(ratelimit_t *ratelimit, msg_t *pMsg, msg_t **ppRepMsg) +{ + int bNeedUnlockMutex = 0; + rsRetVal localRet; + DEFiRet; + + if((pMsg->msgFlags & NEEDS_PARSING) != 0) { + if((localRet = parser.ParseMsg(pMsg)) != RS_RET_OK) { + DBGPRINTF("Message discarded, parsing error %d\n", localRet); + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } + } + + if(ratelimit->bThreadSafe) { + pthread_mutex_lock(&ratelimit->mut); + bNeedUnlockMutex = 1; + } + + if( ratelimit->pMsg != NULL && + getMSGLen(pMsg) == getMSGLen(ratelimit->pMsg) && + !ustrcmp(getMSG(pMsg), getMSG(ratelimit->pMsg)) && + !strcmp(getHOSTNAME(pMsg), getHOSTNAME(ratelimit->pMsg)) && + !strcmp(getPROCID(pMsg, LOCK_MUTEX), getPROCID(ratelimit->pMsg, LOCK_MUTEX)) && + !strcmp(getAPPNAME(pMsg, LOCK_MUTEX), getAPPNAME(ratelimit->pMsg, LOCK_MUTEX))) { + ratelimit->nsupp++; + DBGPRINTF("msg repeated %d times\n", ratelimit->nsupp); + /* use current message, so we have the new timestamp + * (means we need to discard previous one) */ + msgDestruct(&ratelimit->pMsg); + ratelimit->pMsg = pMsg; + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } else {/* new message, do "repeat processing" & save it */ + if(ratelimit->pMsg != NULL) { + if(ratelimit->nsupp > 0) { + *ppRepMsg = ratelimitGenRepMsg(ratelimit); + ratelimit->nsupp = 0; + } + msgDestruct(&ratelimit->pMsg); + } + ratelimit->pMsg = MsgAddRef(pMsg); + } + +finalize_it: + if(bNeedUnlockMutex) + pthread_mutex_unlock(&ratelimit->mut); + RETiRet; +} + + +/* helper: tell how many messages we lost due to linux-like ratelimiting */ +static inline void +tellLostCnt(ratelimit_t *ratelimit) +{ + uchar msgbuf[1024]; + if(ratelimit->missed) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "%s: %u messages lost due to rate-limiting", + ratelimit->name, ratelimit->missed); + ratelimit->missed = 0; + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + } +} + +/* Linux-like ratelimiting, modelled after the linux kernel + * returns 1 if message is within rate limit and shall be + * processed, 0 otherwise. + * This implementation is NOT THREAD-SAFE and must not + * be called concurrently. + */ +static inline int +withinRatelimit(ratelimit_t *ratelimit, time_t tt) +{ + int ret; + uchar msgbuf[1024]; + + if(ratelimit->interval == 0) { + ret = 1; + goto finalize_it; + } + + /* we primarily need "NoTimeCache" mode for imjournal, as it + * sets the message generation time to the journal timestamp. + * As such, we do not get a proper indication of the actual + * message rate. To prevent this, we need to query local + * system time ourselvs. + */ + if(ratelimit->bNoTimeCache) + tt = time(NULL); + + assert(ratelimit->burst != 0); + + if(ratelimit->begin == 0) + ratelimit->begin = tt; + + /* resume if we go out of time window */ + if(tt > ratelimit->begin + ratelimit->interval) { + ratelimit->begin = 0; + ratelimit->done = 0; + tellLostCnt(ratelimit); + } + + /* do actual limit check */ + if(ratelimit->burst > ratelimit->done) { + ratelimit->done++; + ret = 1; + } else { + ratelimit->missed++; + if(ratelimit->missed == 1) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "%s: begin to drop messages due to rate-limiting", + ratelimit->name); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + } + ret = 0; + } + +finalize_it: + return ret; +} + + +/* ratelimit a message, that means: + * - handle "last message repeated n times" logic + * - handle actual (discarding) rate-limiting + * This function returns RS_RET_OK, if the caller shall process + * the message regularly and RS_RET_DISCARD if the caller must + * discard the message. The caller should also discard the message + * if another return status occurs. This places some burden on the + * caller logic, but provides best performance. Demanding this + * cooperative mode can enable a faulty caller to thrash up part + * of the system, but we accept that risk (a faulty caller can + * always do all sorts of evil, so...) + * If *ppRepMsg != NULL on return, the caller must enqueue that + * message before the original message. + */ +rsRetVal +ratelimitMsg(ratelimit_t *ratelimit, msg_t *pMsg, msg_t **ppRepMsg) +{ + DEFiRet; + + *ppRepMsg = NULL; + /* Only the messages having severity level at or below the + * treshold (the value is >=) are subject to ratelimiting. */ + if(ratelimit->interval && (pMsg->iSeverity >= ratelimit->severity)) { + if(withinRatelimit(ratelimit, pMsg->ttGenTime) == 0) { + msgDestruct(&pMsg); + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } + } + if(ratelimit->bReduceRepeatMsgs) { + CHKiRet(doLastMessageRepeatedNTimes(ratelimit, pMsg, ppRepMsg)); + } +finalize_it: + RETiRet; +} + +/* returns 1, if the ratelimiter performs any checks and 0 otherwise */ +int +ratelimitChecked(ratelimit_t *ratelimit) +{ + return ratelimit->interval || ratelimit->bReduceRepeatMsgs; +} + + +/* add a message to a ratelimiter/multisubmit structure. + * ratelimiting is automatically handled according to the ratelimit + * settings. + * if pMultiSub == NULL, a single-message enqueue happens (under reconsideration) + */ +rsRetVal +ratelimitAddMsg(ratelimit_t *ratelimit, multi_submit_t *pMultiSub, msg_t *pMsg) +{ + rsRetVal localRet; + msg_t *repMsg; + DEFiRet; + + if(pMultiSub == NULL) { + localRet = ratelimitMsg(ratelimit, pMsg, &repMsg); + if(repMsg != NULL) + CHKiRet(submitMsg2(repMsg)); + if(localRet == RS_RET_OK) + CHKiRet(submitMsg2(pMsg)); + } else { + localRet = ratelimitMsg(ratelimit, pMsg, &repMsg); + if(repMsg != NULL) { + pMultiSub->ppMsgs[pMultiSub->nElem++] = repMsg; + if(pMultiSub->nElem == pMultiSub->maxElem) + CHKiRet(multiSubmitMsg2(pMultiSub)); + } + if(localRet == RS_RET_OK) { + pMultiSub->ppMsgs[pMultiSub->nElem++] = pMsg; + if(pMultiSub->nElem == pMultiSub->maxElem) + CHKiRet(multiSubmitMsg2(pMultiSub)); + } + } + +finalize_it: + RETiRet; +} + + +/* modname must be a static name (usually expected to be the module + * name and MUST be present. dynname may be NULL and can be used for + * dynamic information, e.g. PID or listener IP, ... + * Both values should be kept brief. + */ +rsRetVal +ratelimitNew(ratelimit_t **ppThis, char *modname, char *dynname) +{ + ratelimit_t *pThis; + char namebuf[256]; + DEFiRet; + + CHKmalloc(pThis = calloc(1, sizeof(ratelimit_t))); + if(modname == NULL) + modname ="*ERROR:MODULE NAME MISSING*"; + + if(dynname == NULL) { + pThis->name = strdup(modname); + } else { + snprintf(namebuf, sizeof(namebuf), "%s[%s]", + modname, dynname); + namebuf[sizeof(namebuf)-1] = '\0'; /* to be on safe side */ + pThis->name = strdup(namebuf); + } + /* pThis->severity == 0 - all messages are ratelimited */ + pThis->bReduceRepeatMsgs = loadConf->globals.bReduceRepeatMsgs; + *ppThis = pThis; +finalize_it: + RETiRet; +} + + +/* enable linux-like ratelimiting */ +void +ratelimitSetLinuxLike(ratelimit_t *ratelimit, unsigned short interval, unsigned short burst) +{ + ratelimit->interval = interval; + ratelimit->burst = burst; + ratelimit->done = 0; + ratelimit->missed = 0; + ratelimit->begin = 0; +} + + +/* enable thread-safe operations mode. This make sure that + * a single ratelimiter can be called from multiple threads. As + * this causes some overhead and is not always required, it needs + * to be explicitely enabled. This operation cannot be undone + * (think: why should one do that???) + */ +void +ratelimitSetThreadSafe(ratelimit_t *ratelimit) +{ + ratelimit->bThreadSafe = 1; + pthread_mutex_init(&ratelimit->mut, NULL); +} +void +ratelimitSetNoTimeCache(ratelimit_t *ratelimit) +{ + ratelimit->bNoTimeCache = 1; + pthread_mutex_init(&ratelimit->mut, NULL); +} + +/* Severity level determines which messages are subject to + * ratelimiting. Default (no value set) is all messages. + */ +void +ratelimitSetSeverity(ratelimit_t *ratelimit, intTiny severity) +{ + ratelimit->severity = severity; +} + +void +ratelimitDestruct(ratelimit_t *ratelimit) +{ + msg_t *pMsg; + if(ratelimit->pMsg != NULL) { + if(ratelimit->nsupp > 0) { + pMsg = ratelimitGenRepMsg(ratelimit); + if(pMsg != NULL) + submitMsg2(pMsg); + } + msgDestruct(&ratelimit->pMsg); + } + tellLostCnt(ratelimit); + if(ratelimit->bThreadSafe) + pthread_mutex_destroy(&ratelimit->mut); + free(ratelimit->name); + free(ratelimit); +} + +void +ratelimitModExit(void) +{ + objRelease(datetime, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); +} + +rsRetVal +ratelimitModInit(void) +{ + DEFiRet; + CHKiRet(objGetObjInterface(&obj)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); +finalize_it: + RETiRet; +} diff --git a/runtime/ratelimit.h b/runtime/ratelimit.h new file mode 100644 index 00000000..563777fd --- /dev/null +++ b/runtime/ratelimit.h @@ -0,0 +1,55 @@ +/* header for ratelimit.c + * + * Copyright 2012 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_RATELIMIT_H +#define INCLUDED_RATELIMIT_H + +struct ratelimit_s { + char *name; /**< rate limiter name, e.g. for user messages */ + /* support for Linux kernel-type ratelimiting */ + unsigned short interval; + unsigned short burst; + intTiny severity; /**< ratelimit only equal or lower severity levels (eq or higher values) */ + unsigned done; + unsigned missed; + time_t begin; + /* support for "last message repeated n times */ + int bReduceRepeatMsgs; /**< shall we do "last message repeated n times" processing? */ + unsigned nsupp; /**< nbr of msgs suppressed */ + msg_t *pMsg; + sbool bThreadSafe; /**< do we need to operate in Thread-Safe mode? */ + sbool bNoTimeCache; /**< if we shall not used cached reception time */ + pthread_mutex_t mut; /**< mutex if thread-safe operation desired */ +}; + +/* prototypes */ +rsRetVal ratelimitNew(ratelimit_t **ppThis, char *modname, char *dynname); +void ratelimitSetThreadSafe(ratelimit_t *ratelimit); +void ratelimitSetLinuxLike(ratelimit_t *ratelimit, unsigned short interval, unsigned short burst); +void ratelimitSetNoTimeCache(ratelimit_t *ratelimit); +void ratelimitSetSeverity(ratelimit_t *ratelimit, intTiny severity); +rsRetVal ratelimitMsg(ratelimit_t *ratelimit, msg_t *pMsg, msg_t **ppRep); +rsRetVal ratelimitAddMsg(ratelimit_t *ratelimit, multi_submit_t *pMultiSub, msg_t *pMsg); +void ratelimitDestruct(ratelimit_t *pThis); +int ratelimitChecked(ratelimit_t *ratelimit); +rsRetVal ratelimitModInit(void); +void ratelimitModExit(void); + +#endif /* #ifndef INCLUDED_RATELIMIT_H */ diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 118e9c11..d8b81f1b 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -36,7 +36,6 @@ #include "rsyslog.h" #include "obj.h" #include "srUtils.h" -#include "rule.h" #include "ruleset.h" #include "modules.h" #include "conf.h" @@ -68,9 +67,9 @@ #include "dirty.h" #include "template.h" +extern char* yytext; /* static data */ DEFobjStaticHelpers -DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) DEFobjCurrIf(module) DEFobjCurrIf(conf) @@ -116,8 +115,8 @@ void cnfDoCfsysline(char *ln); */ BEGINobjConstruct(rsconf) /* be sure to specify the object type also in END macro! */ pThis->globals.bDebugPrintTemplateList = 1; - pThis->globals.bDebugPrintModuleList = 1; - pThis->globals.bDebugPrintCfSysLineHandlerList = 1; + pThis->globals.bDebugPrintModuleList = 0; + pThis->globals.bDebugPrintCfSysLineHandlerList = 0; pThis->globals.bLogStatusMsgs = DFLT_bLogStatusMsgs; pThis->globals.bErrMsgToStderr = 1; pThis->globals.umask = -1; @@ -254,54 +253,6 @@ CODESTARTobjDebugPrint(rsconf) ENDobjDebugPrint(rsconf) -rsRetVal -cnfDoActlst(struct cnfactlst *actlst, rule_t *pRule) -{ - struct cnfcfsyslinelst *cflst; - action_t *pAction; - uchar *str; - rsRetVal localRet; - DEFiRet; - - while(actlst != NULL) { - dbgprintf("aclst %p: ", actlst); - if(actlst->actType == CNFACT_V2) { - dbgprintf("v6+ action object\n"); - if(actionNewInst(actlst->data.lst, &pAction) == RS_RET_OK) { - iRet = llAppend(&(pRule)->llActList, NULL, (void*) pAction); - } else { - errmsg.LogError(0, RS_RET_ERR, "errors occured in file '%s' " - "around line %d", actlst->cnfFile, actlst->lineno); - } - } else { - DBGPRINTF("legacy action line:%s\n", actlst->data.legActLine); - str = (uchar*) actlst->data.legActLine; - if((localRet = cflineDoAction(loadConf, &str, &pAction)) != RS_RET_OK) { - uchar szErrLoc[MAXFNAME + 64]; - if(localRet != RS_RET_OK_WARN) { - DBGPRINTF("legacy action line NOT successfully processed\n"); - } - snprintf((char*)szErrLoc, sizeof(szErrLoc) / sizeof(uchar), - "%s, line %d", actlst->cnfFile, actlst->lineno); - errmsg.LogError(0, NO_ERRCODE, "the last %s occured in %s:\"%s\"", - (localRet == RS_RET_OK_WARN) ? "warning" : "error", - (char*)szErrLoc, (char*)actlst->data.legActLine); - if(localRet != RS_RET_OK_WARN) { - ABORT_FINALIZE(localRet); - } - } - iRet = llAppend(&(pRule)->llActList, NULL, (void*) pAction); - } - for( cflst = actlst->syslines - ; cflst != NULL ; cflst = cflst->next) { - cnfDoCfsysline(cflst->line); - } - actlst = actlst->next; - } -finalize_it: - RETiRet; -} - /* This function returns the current date in different * variants. It is used to construct the $NOW series of * system properties. The returned buffer must be freed @@ -341,6 +292,9 @@ getNOW(eNOWType eNow, es_str_t **estr) case NOW_MINUTE: len = snprintf((char*) szBuf, sizeof(szBuf)/sizeof(uchar), "%2.2d", t.minute); break; + default: + len = snprintf((char*) szBuf, sizeof(szBuf)/sizeof(uchar), "*invld eNow*"); + break; } /* now create a string object out of it and hand that over to the var */ @@ -397,15 +351,11 @@ inputProcessCnf(struct cnfobj *o) pvals = nvlstGetParams(o->nvlst, &inppblk, NULL); if(pvals == NULL) { - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } DBGPRINTF("input param blk after inputProcessCnf:\n"); cnfparamsPrint(&inppblk, pvals); typeIdx = cnfparamGetIdx(&inppblk, "type"); - if(pvals[typeIdx].bUsed == 0) { - errmsg.LogError(0, RS_RET_CONF_RQRD_PARAM_MISSING, "input type missing"); - ABORT_FINALIZE(RS_RET_CONF_RQRD_PARAM_MISSING); // TODO: move this into rainerscript handlers - } cnfModName = (uchar*)es_str2cstr(pvals[typeIdx].val.d.estr, NULL); if((pMod = module.FindWithCnfName(loadConf, cnfModName, eMOD_IN)) == NULL) { errmsg.LogError(0, RS_RET_MOD_UNKNOWN, "input module name '%s' is unknown", cnfModName); @@ -435,9 +385,6 @@ parser_errmsg(char *fmt, ...) va_start(ap, fmt); if(vsnprintf(errBuf, sizeof(errBuf), fmt, ap) == sizeof(errBuf)) errBuf[sizeof(errBuf)-1] = '\0'; -dbgprintf("XXXX: msg: %s\n", errBuf); -dbgprintf("XXXX: cnfcurrfn: %s\n", cnfcurrfn); -dbgprintf("XXXX: yylineno: %d\n", yylineno); errmsg.LogError(0, RS_RET_CONF_PARSE_ERROR, "error during parsing file %s, on or before line %d: %s", cnfcurrfn, yylineno, errBuf); @@ -447,7 +394,7 @@ dbgprintf("XXXX: yylineno: %d\n", yylineno); int yyerror(char *s) { - parser_errmsg("%s", s); + parser_errmsg("%s on token '%s'", s, yytext); return 0; } void cnfDoObj(struct cnfobj *o) @@ -463,75 +410,35 @@ void cnfDoObj(struct cnfobj *o) case CNFOBJ_MODULE: modulesProcessCnf(o); break; - case CNFOBJ_ACTION: - actionProcessCnf(o); - break; case CNFOBJ_INPUT: inputProcessCnf(o); break; case CNFOBJ_TPL: - tplProcessCnf(o); + if(tplProcessCnf(o) != RS_RET_OK) + parser_errmsg("error processing template object"); + break; + case CNFOBJ_RULESET: + rulesetProcessCnf(o); break; case CNFOBJ_PROPERTY: case CNFOBJ_CONSTANT: /* these types are processed at a later stage */ bChkUnuse = 0; break; + default: + dbgprintf("cnfDoObj program error: unexpected object type %u\n", + o->objType); + break; } if(bChkUnuse) nvlstChkUnused(o->nvlst); cnfobjDestruct(o); } -void cnfDoRule(struct cnfrule *cnfrule) +void cnfDoScript(struct cnfstmt *script) { - rule_t *pRule; - uchar *str; - rsRetVal iRet = RS_RET_OK; //DEFiRet; - - dbgprintf("cnf:global:rule\n"); - cnfrulePrint(cnfrule); - - CHKiRet(rule.Construct(&pRule)); /* create "fresh" selector */ - CHKiRet(rule.SetAssRuleset(pRule, ruleset.GetCurrent(loadConf))); - CHKiRet(rule.ConstructFinalize(pRule)); - - switch(cnfrule->filttype) { - case CNFFILT_NONE: - break; - case CNFFILT_PRI: - str = (uchar*) cnfrule->filt.s; - iRet = cflineProcessTradPRIFilter(&str, pRule); - break; - case CNFFILT_PROP: - dbgprintf("%s\n", cnfrule->filt.s); - str = (uchar*) cnfrule->filt.s; - iRet = cflineProcessPropFilter(&str, pRule); - break; - case CNFFILT_SCRIPT: - pRule->f_filter_type = FILTER_EXPR; - pRule->f_filterData.expr = cnfrule->filt.expr; - break; - } - /* we now check if there are some global (BSD-style) filter conditions - * and, if so, we copy them over. rgerhards, 2005-10-18 - */ - if(pDfltProgNameCmp != NULL) { - CHKiRet(rsCStrConstructFromCStr(&(pRule->pCSProgNameComp), pDfltProgNameCmp)); - } - - if(eDfltHostnameCmpMode != HN_NO_COMP) { - pRule->eHostnameCmpMode = eDfltHostnameCmpMode; - CHKiRet(rsCStrConstructFromCStr(&(pRule->pCSHostnameComp), pDfltHostnameCmp)); - } - - cnfDoActlst(cnfrule->actlst, pRule); - - CHKiRet(ruleset.AddRule(rule.GetAssRuleset(pRule), &pRule)); - -finalize_it: - //TODO: do something with error states - cnfruleDestruct(cnfrule); + dbgprintf("cnf:global:script\n"); + ruleset.AddScript(ruleset.GetCurrent(loadConf), script); } void cnfDoCfsysline(char *ln) @@ -545,13 +452,21 @@ void cnfDoCfsysline(char *ln) void cnfDoBSDTag(char *ln) { DBGPRINTF("cnf:global:BSD tag: %s\n", ln); - cflineProcessTagSelector((uchar**)&ln); + errmsg.LogError(0, RS_RET_BSD_BLOCKS_UNSUPPORTED, + "BSD-style blocks are no longer supported in rsyslog, " + "see http://www.rsyslog.com/g/BSD for details and a " + "solution (Block '%s')", ln); + free(ln); } void cnfDoBSDHost(char *ln) { DBGPRINTF("cnf:global:BSD host: %s\n", ln); - cflineProcessHostSelector((uchar**)&ln); + errmsg.LogError(0, RS_RET_BSD_BLOCKS_UNSUPPORTED, + "BSD-style blocks are no longer supported in rsyslog, " + "see http://www.rsyslog.com/g/BSD for details and a " + "solution (Block '%s')", ln); + free(ln); } es_str_t* @@ -565,6 +480,9 @@ cnfGetVar(char *name, void *usrptr) estr = msgGetCEEVarNew((msg_t*) usrptr, name+2); else estr = msgGetMsgVarNew((msg_t*) usrptr, (uchar*)name+1); + } else { /* if this happens, we have a program logic error */ + estr = es_newStrFromCStr("err: var must start with $", + strlen("err: var must start with $")); } if(Debug) { char *s; @@ -841,7 +759,7 @@ activateMainQueue() { DEFiRet; /* create message queue */ - CHKiRet_Hdlr(createMainQueue(&pMsgQueue, UCHAR_CONSTANT("main Q"))) { + CHKiRet_Hdlr(createMainQueue(&pMsgQueue, UCHAR_CONSTANT("main Q"), NULL)) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); FINALIZE; @@ -961,6 +879,7 @@ setCurrRuleset(void __attribute__((unused)) *pVal, uchar *pszName) CHKiRet(ruleset.Construct(&pRuleset)); CHKiRet(ruleset.SetName(pRuleset, pszName)); CHKiRet(ruleset.ConstructFinalize(ourConf, pRuleset)); + rulesetSetCurrRulesetPtr(pRuleset); } else { ABORT_FINALIZE(localRet); } @@ -1162,6 +1081,7 @@ initLegacyConf(void) ruleset.Construct(&pRuleset); ruleset.SetName(pRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); ruleset.ConstructFinalize(loadConf, pRuleset); + rulesetSetCurrRulesetPtr(pRuleset); /* now register config handlers */ CHKiRet(regCfSysLineHdlr((uchar *)"sleep", 0, eCmdHdlrGoneAway, @@ -1378,6 +1298,7 @@ ourConf = loadConf; // TODO: remove, once ourConf is gone! ABORT_FINALIZE(RS_RET_NO_ACTIONS); } tellLexEndParsing(); + rulesetOptimizeAll(loadConf); tellCoreConfigLoadDone(); tellModulesConfigLoadDone(); @@ -1436,7 +1357,6 @@ ENDobjQueryInterface(rsconf) BEGINObjClassInit(rsconf, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(ruleset, CORE_COMPONENT)); - CHKiRet(objUse(rule, CORE_COMPONENT)); CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); @@ -1453,7 +1373,6 @@ ENDObjClassInit(rsconf) /* De-initialize the rsconf class. */ BEGINObjClassExit(rsconf, OBJ_IS_CORE_MODULE) /* class, version */ - objRelease(rule, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); objRelease(module, CORE_COMPONENT); objRelease(conf, CORE_COMPONENT); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index cbab06b7..047dfa9b 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -72,7 +72,6 @@ #include "glbl.h" #include "errmsg.h" #include "prop.h" -#include "rule.h" #include "ruleset.h" #include "parser.h" #include "strgen.h" @@ -171,8 +170,6 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(glblClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "rule"; - CHKiRet(ruleClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ruleset"; CHKiRet(rulesetClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "wti"; @@ -220,7 +217,6 @@ rsrtExit(void) confClassExit(); glblClassExit(); rulesetClassExit(); - ruleClassExit(); objClassExit(); /* *THIS* *MUST/SHOULD?* always be the first class initilizer being called (except debug)! */ } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index a6e4b100..47b34783 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -46,6 +46,7 @@ #define CONF_HOSTNAME_MAXSIZE 512 /* a value that is deemed far too large for any valid HOSTNAME */ #define CONF_RAWMSG_BUFSIZE 101 #define CONF_TAG_BUFSIZE 32 +#define CONF_PROGNAME_BUFSIZE 16 #define CONF_HOSTNAME_BUFSIZE 32 #define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ #define CONF_MIN_SIZE_FOR_COMPRESS 60 /* config param: minimum message size to try compression. The smaller @@ -60,15 +61,16 @@ * rgerhards, 2006-11-30 */ -#define CONF_OMOD_NUMSTRINGS_MAXSIZE 3 /* cache for pointers to output module buffer pointers. All - * rsyslog-provided plugins do NOT need more than three buffers. If - * more are needed (future developments, third-parties), rsyslog +#define CONF_OMOD_NUMSTRINGS_MAXSIZE 5 /* cache for pointers to output module buffer pointers. All + * rsyslog-provided plugins do NOT need more than five buffers. If + * more are needed (future developments, third-parties), rsyslog * must be recompiled with a larger parameter. Hardcoding this * saves us some overhead, both in runtime in code complexity. As * it is doubtful if ever more than 3 parameters are needed, the * approach taken here is considered appropriate. * rgerhards, 2010-06-24 */ +#define CONF_NUM_MULTISUB 1024 /* default number of messages per multisub structure */ /* ############################################################# * * # End Config Settings # * @@ -89,7 +91,7 @@ /* the rsyslog core provides information about present feature to plugins - * asking it. Below are feature-test macros which must be used to query + * asking it. Below are feature-test macros which must be used to query * features. Note that this must be powers of two, so that multiple queries * can be combined. -- rgerhards, 2009-04-27 */ @@ -151,7 +153,7 @@ typedef uintTiny propid_t; */ enum rsRetVal_ /** return value. All methods return this if not specified otherwise */ { - /* the first two define are for errmsg.logError(), so that we can use the rsRetVal + /* the first two define are for errmsg.logError(), so that we can use the rsRetVal * as an rsyslog error code. -- rgerhards, 20080-06-27 */ RS_RET_NO_ERRCODE = -1, /**< RESERVED for NO_ERRCODE errmsg.logError status name */ @@ -322,7 +324,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_DOOR = -2147, /**< some problems with handling the Solaris door functionality */ RS_RET_NO_SRCNAME_TPL = -2150, /**< sourcename template was not specified where one was needed (omudpspoof spoof addr) */ RS_RET_HOST_NOT_SPECIFIED = -2151, /**< (target) host was not specified where it was needed */ - RS_RET_ERR_LIBNET_INIT = -2152, /**< error initializing libnet */ + RS_RET_ERR_LIBNET_INIT = -2152, /**< error initializing libnet, e.g. because not running as root */ RS_RET_FORCE_TERM = -2153, /**< thread was forced to terminate by bShallShutdown, a state, not an error */ RS_RET_RULES_QUEUE_EXISTS = -2154,/**< we were instructed to create a new ruleset queue, but one already exists */ RS_RET_NO_CURR_RULESET = -2155,/**< no current ruleset exists (but one is required) */ @@ -376,15 +378,45 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_LEGA_ACT_NOT_SUPPORTED = -2215, /**< the module (no longer) supports legacy action syntax */ RS_RET_MAX_OMSR_REACHED = -2216, /**< max nbr of string requests reached, not supported by core */ RS_RET_UID_MISSING = -2217, /**< a user id is missing (but e.g. a password provided) */ + RS_RET_DATAFAIL = -2218, /**< data passed to action caused failure */ /* reserved for pre-v6.5 */ RS_RET_DUP_PARAM = -2220, /**< config parameter is given more than once */ RS_RET_MODULE_ALREADY_IN_CONF = -2221, /**< module already in current configuration */ RS_RET_PARAM_NOT_PERMITTED = -2222, /**< legacy parameter no longer permitted (usally already set by v2) */ RS_RET_NO_JSON_PASSING = -2223, /**< rsyslog core does not support JSON-passing plugin API */ RS_RET_MOD_NO_INPUT_STMT = -2224, /**< (input) module does not support input() statement */ + RS_RET_NO_CEE_MSG = -2225, /**< the message being processed is NOT CEE-enhanced */ + + /**** up to 2290 is reserved for v6 use ****/ + RS_RET_RELP_ERR = -2291, /**<< error in RELP processing */ + /**** up to 3000 is reserved for c7 use ****/ + RS_RET_JNAME_NO_ROOT = -2301, /**< root element is missing in JSON path */ + RS_RET_JNAME_INVALID = -2302, /**< JSON path is invalid */ + RS_RET_JSON_PARSE_ERR = -2303, /**< we had a problem parsing JSON (or extra data) */ + RS_RET_BSD_BLOCKS_UNSUPPORTED = -2304, /**< BSD-style config blocks are no longer supported */ + RS_RET_JNAME_NOTFOUND = -2305, /**< JSON name not found (does not exist) */ + RS_RET_INVLD_SETOP = -2305, /**< invalid variable set operation, incompatible type */ + RS_RET_RULESET_EXISTS = -2306,/**< ruleset already exists */ + RS_RET_DEPRECATED = -2307,/**< deprecated functionality is used */ + RS_RET_DS_PROP_SEQ_ERR = -2308,/**< property sequence error deserializing object */ + RS_RET_TPL_INVLD_PROP = -2309,/**< property name error in template (unknown name) */ + RS_RET_NO_RULEBASE = -2310,/**< mmnormalize: rulebase can not be found or otherwise invalid */ + RS_RET_INVLD_MODE = -2311,/**< invalid mode specified in configuration */ + RS_RET_INVLD_ANON_BITS = -2312,/**< mmanon: invalid number of bits to anonymize specified */ + RS_RET_REPLCHAR_IGNORED = -2313,/**< mmanon: replacementChar parameter is ignored */ + RS_RET_SIGPROV_ERR = -2320,/**< error in signature provider */ + RS_RET_CRYPROV_ERR = -2321,/**< error in cryptography encryption provider */ + RS_RET_EI_OPN_ERR = -2322,/**< error opening an .encinfo file */ + RS_RET_EI_NO_EXISTS = -2323,/**< .encinfo file does not exist (status, not necessarily error!)*/ + RS_RET_EI_WR_ERR = -2324,/**< error writing an .encinfo file */ + RS_RET_EI_INVLD_FILE = -2325,/**< header indicates the file is no .encinfo file */ + RS_RET_CRY_INVLD_ALGO = -2326,/**< user specified invalid (unkonwn) crypto algorithm */ + RS_RET_CRY_INVLD_MODE = -2327,/**< user specified invalid (unkonwn) crypto mode */ + RS_RET_QUEUE_DISK_NO_FN = -2328,/**< disk queue configured, but filename not set */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ + RS_RET_FIELD_NOT_FOUND = 1002, /**< field() function did not find requested field */ /* some generic error/status codes */ RS_RET_OK = 0, /**< operation successful */ @@ -417,7 +449,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth /** Object ID. These are for internal checking. Each * object is assigned a specific ID. This is contained in - * all Object structs (just like C++ RTTI). We can use + * all Object structs (just like C++ RTTI). We can use * this field to see if we have been passed a correct ID. * Other than that, there is currently no other use for * the object id. @@ -449,7 +481,7 @@ typedef enum rsObjectID rsObjID; #endif /** - * This macro should be used to free objects. + * This macro should be used to free objects. * It aids in interpreting dumps during debugging. */ #ifdef NDEBUG @@ -516,7 +548,7 @@ rsRetVal rsrtSetErrLogger(rsRetVal (*errLogger)(int, uchar*)); /* TODO: remove this -- this is only for transition of the config system */ extern rsconf_t *ourConf; /* defined by syslogd.c, a hack for functions that do not - yet receive a copy, so that we can incrementially + yet receive a copy, so that we can incrementially compile and change... -- rgerhars, 2011-04-19 */ #endif /* multi-include protection */ diff --git a/runtime/rule.c b/runtime/rule.c deleted file mode 100644 index fc1e740f..00000000 --- a/runtime/rule.c +++ /dev/null @@ -1,479 +0,0 @@ -/* rule.c - rsyslog's rule object - * - * See file comment in rule.c for the overall structure of rule processing. - * - * Module begun 2009-06-10 by Rainer Gerhards - * - * Copyright 2009-2012 Adiscon GmbH. - * - * This file is part of the rsyslog runtime library. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * -or- - * see COPYING.ASL20 in the source distribution - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "config.h" -#include <stdlib.h> -#include <string.h> -#include <assert.h> -#include <ctype.h> - -#include "rsyslog.h" -#include "obj.h" -#include "action.h" -#include "rule.h" -#include "errmsg.h" -#include "srUtils.h" -#include "batch.h" -#include "parserif.h" -#include "unicode-helper.h" - -/* static data */ -DEFobjStaticHelpers -DEFobjCurrIf(errmsg) - - -/* support for simple textual representation of FIOP names - * rgerhards, 2005-09-27 - */ -static char* -getFIOPName(unsigned iFIOP) -{ - char *pRet; - switch(iFIOP) { - case FIOP_CONTAINS: - pRet = "contains"; - break; - case FIOP_ISEQUAL: - pRet = "isequal"; - break; - case FIOP_STARTSWITH: - pRet = "startswith"; - break; - case FIOP_REGEX: - pRet = "regex"; - break; - case FIOP_EREREGEX: - pRet = "ereregex"; - break; - case FIOP_ISEMPTY: - pRet = "isempty"; - break; - default: - pRet = "NOP"; - break; - } - return pRet; -} - - -/* iterate over all actions, this is often needed, for example when HUP processing - * must be done or a shutdown is pending. - */ -static rsRetVal -iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) -{ - return llExecFunc(&pThis->llActList, pFunc, pParam); -} - - -/* helper to processMsg(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(processBatchDoActions) -{ - DEFiRet; - rsRetVal iRetMod; /* return value of module - we do not always pass that back */ - action_t *pAction = (action_t*) pData; - batch_t *pBatch = (batch_t*) pParam; - - DBGPRINTF("Processing next action\n"); - iRetMod = pAction->submitToActQ(pAction, pBatch); - - RETiRet; -} - - -/* This functions looks at the given message and checks if it matches the - * provided filter condition. - */ -static rsRetVal -shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, sbool *bProcessMsg) -{ - DEFiRet; - unsigned short pbMustBeFreed; - uchar *pszPropVal; - int bRet = 0; - size_t propLen; - - ISOBJ_TYPE_assert(pRule, rule); - assert(pMsg != NULL); - - /* we first have a look at the global, BSD-style block filters (for tag - * and host). Only if they match, we evaluate the actual filter. - * rgerhards, 2005-10-18 - */ - if(pRule->eHostnameCmpMode == HN_NO_COMP) { - /* EMPTY BY INTENSION - we check this value first, because - * it is the one most often used, so this saves us time! - */ - } else if(pRule->eHostnameCmpMode == HN_COMP_MATCH) { - if(rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - DBGPRINTF("hostname filter '+%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } else { /* must be -hostname */ - if(!rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, SO WE ARe already done... */ - DBGPRINTF("hostname filter '-%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } - - if(pRule->pCSProgNameComp != NULL) { - int bInv = 0, bEqv = 0, offset = 0; - if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp)) == '-') { - if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp) + 1) == '-') - offset = 1; - else { - bInv = 1; - offset = 1; - } - } - if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, - (uchar*) getProgramName(pMsg, LOCK_MUTEX), getProgramNameLen(pMsg, LOCK_MUTEX))) - bEqv = 1; - - if((!bEqv && !bInv) || (bEqv && bInv)) { - /* not equal or inverted selection, so we are already done... */ - DBGPRINTF("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg, LOCK_MUTEX)); - FINALIZE; - } - } - - /* done with the BSD-style block filters */ - - if(pRule->f_filter_type == FILTER_PRI) { - /* skip messages that are incorrect priority */ - if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ - ((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - DBGPRINTF("testing filter, f_pmask %d, result %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility], bRet); - } else if(pRule->f_filter_type == FILTER_EXPR) { - bRet = cnfexprEvalBool(pRule->f_filterData.expr, pMsg); - DBGPRINTF("result of rainerscript filter evaluation: %d\n", bRet); - } else { - assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ - if(pRule->f_filterData.prop.propID == PROP_INVALID) { - DBGPRINTF("invalid property ID, filter always returns 0\n"); - bRet = 0; - } else { - pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.propID, - pRule->f_filterData.prop.propName, &propLen, &pbMustBeFreed); - - /* Now do the compares (short list currently ;)) */ - switch(pRule->f_filterData.prop.operation ) { - case FIOP_CONTAINS: - if(rsCStrLocateInSzStr(pRule->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) - bRet = 1; - break; - case FIOP_ISEMPTY: - if(propLen == 0) - bRet = 1; /* process message! */ - break; - case FIOP_ISEQUAL: - if(rsCStrSzStrCmp(pRule->f_filterData.prop.pCSCompValue, - pszPropVal, ustrlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_STARTSWITH: - if(rsCStrSzStrStartsWithCStr(pRule->f_filterData.prop.pCSCompValue, - pszPropVal, ustrlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - case FIOP_EREREGEX: - if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - default: - /* here, it handles NOP (for performance reasons) */ - assert(pRule->f_filterData.prop.operation == FIOP_NOP); - bRet = 1; /* as good as any other default ;) */ - break; - } - - /* now check if the value must be negated */ - if(pRule->f_filterData.prop.isNegated) - bRet = (bRet == 1) ? 0 : 1; - - if(Debug) { - char *cstr; - if(pRule->f_filterData.prop.propID == PROP_CEE) { - cstr = es_str2cstr(pRule->f_filterData.prop.propName, NULL); - dbgprintf("Filter: check for CEE property '%s' (value '%s') ", - cstr, pszPropVal); - free(cstr); - } else { - dbgprintf("Filter: check for property '%s' (value '%s') ", - propIDToName(pRule->f_filterData.prop.propID), pszPropVal); - } - if(pRule->f_filterData.prop.isNegated) - dbgprintf("NOT "); - if(pRule->f_filterData.prop.operation == FIOP_ISEMPTY) { - dbgprintf("%s : %s\n", - getFIOPName(pRule->f_filterData.prop.operation), - bRet ? "TRUE" : "FALSE"); - } else { - dbgprintf("%s '%s': %s\n", - getFIOPName(pRule->f_filterData.prop.operation), - rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue), - bRet ? "TRUE" : "FALSE"); - } - } - - /* cleanup */ - if(pbMustBeFreed) - free(pszPropVal); - } - } - -finalize_it: - *bProcessMsg = bRet; - RETiRet; -} - - - -/* Process (consume) a batch of messages. Calls the actions configured. - * rgerhards, 2005-10-13 - */ -static rsRetVal -processBatch(rule_t *pThis, batch_t *pBatch) -{ - int i; - rsRetVal localRet; - DEFiRet; - - ISOBJ_TYPE_assert(pThis, rule); - assert(pBatch != NULL); - - /* first check the filters and reset status variables */ - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - localRet = shouldProcessThisMessage(pThis, (msg_t*)(pBatch->pElem[i].pUsrp), - &(pBatch->pElem[i].bFilterOK)); - if(localRet != RS_RET_OK) { - DBGPRINTF("processBatch: iRet %d returned from shouldProcessThisMessage, " - "ignoring message\n", localRet); - pBatch->pElem[i].bFilterOK = 0; - } - if(pBatch->pElem[i].bFilterOK) { - /* re-init only when actually needed (cache write cost!) */ - pBatch->pElem[i].bPrevWasSuspended = 0; - } - } - CHKiRet(llExecFunc(&pThis->llActList, processBatchDoActions, pBatch)); - -finalize_it: - RETiRet; -} - - -/* Standard-Constructor - */ -BEGINobjConstruct(rule) /* be sure to specify the object type also in END macro! */ -ENDobjConstruct(rule) - - -/* ConstructionFinalizer - * rgerhards, 2008-01-09 - */ -static rsRetVal -ruleConstructFinalize(rule_t *pThis) -{ - DEFiRet; - ISOBJ_TYPE_assert(pThis, rule); - - /* note: actionDestruct is from action.c API! */ - CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); - -finalize_it: - RETiRet; -} - - -/* destructor for the rule object */ -BEGINobjDestruct(rule) /* be sure to specify the object type also in END and CODESTART macros! */ -CODESTARTobjDestruct(rule) - if(pThis->pCSHostnameComp != NULL) - rsCStrDestruct(&pThis->pCSHostnameComp); - if(pThis->pCSProgNameComp != NULL) - rsCStrDestruct(&pThis->pCSProgNameComp); - - if(pThis->f_filter_type == FILTER_PROP) { - if(pThis->f_filterData.prop.pCSCompValue != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); - if(pThis->f_filterData.prop.regex_cache != NULL) - rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); - if(pThis->f_filterData.prop.propName != NULL) - es_deleteStr(pThis->f_filterData.prop.propName); - } else if(pThis->f_filter_type == FILTER_EXPR) { - cnfexprDestruct(pThis->f_filterData.expr); - } - - llDestroy(&pThis->llActList); -ENDobjDestruct(rule) - - -/* set the associated ruleset */ -static rsRetVal -setAssRuleset(rule_t *pThis, ruleset_t *pRuleset) -{ - DEFiRet; - ISOBJ_TYPE_assert(pThis, rule); - ISOBJ_TYPE_assert(pRuleset, ruleset); - pThis->pRuleset = pRuleset; - RETiRet; -} - -/* get the associated ruleset (may be NULL if not set!) */ -static ruleset_t* -getAssRuleset(rule_t *pThis) -{ - ISOBJ_TYPE_assert(pThis, rule); - return pThis->pRuleset; -} - - -/* helper to DebugPrint, to print out all actions via - * the llExecFunc() facility. - */ -DEFFUNC_llExecFunc(dbgPrintInitInfoAction) -{ - DEFiRet; - iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); - RETiRet; -} - - -/* debugprint for the rule object */ -BEGINobjDebugPrint(rule) /* be sure to specify the object type also in END and CODESTART macros! */ - int i; - char *cstr; -CODESTARTobjDebugPrint(rule) - dbgoprint((obj_t*) pThis, "rsyslog rule:\n"); - if(pThis->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(pThis->pCSProgNameComp)); - if(pThis->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", - pThis->eHostnameCmpMode == HN_COMP_MATCH ? - "only" : "allbut", - rsCStrGetSzStrNoNULL(pThis->pCSHostnameComp)); - if(pThis->f_filter_type == FILTER_PRI) { - for (i = 0; i <= LOG_NFACILITIES; i++) - if (pThis->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", pThis->f_filterData.f_pmask[i]); - } else if(pThis->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); - } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", propIDToName(pThis->f_filterData.prop.propID)); - if(pThis->f_filterData.prop.propID != PROP_INVALID) { - if(pThis->f_filterData.prop.propName != NULL) { - cstr = es_str2cstr(pThis->f_filterData.prop.propName, NULL); - dbgprintf("\tCEE-Prop.: '%s'\n", cstr); - free(cstr); - } - dbgprintf("\tOperation: "); - if(pThis->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(pThis->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", - rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue)); - } - dbgprintf("\tAction...: "); - } - - dbgprintf("\nActions:\n"); - llExecFunc(&pThis->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - - dbgprintf("\n"); -ENDobjDebugPrint(rule) - - -/* queryInterface function - * rgerhards, 2008-02-21 - */ -BEGINobjQueryInterface(rule) -CODESTARTobjQueryInterface(rule) - if(pIf->ifVersion != ruleCURR_IF_VERSION) { /* check for current version, increment on each change */ - ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); - } - - /* ok, we have the right interface, so let's fill it - * Please note that we may also do some backwards-compatibility - * work here (if we can support an older interface version - that, - * of course, also affects the "if" above). - */ - pIf->Construct = ruleConstruct; - pIf->ConstructFinalize = ruleConstructFinalize; - pIf->Destruct = ruleDestruct; - pIf->DebugPrint = ruleDebugPrint; - - pIf->IterateAllActions = iterateAllActions; - pIf->ProcessBatch = processBatch; - pIf->SetAssRuleset = setAssRuleset; - pIf->GetAssRuleset = getAssRuleset; -finalize_it: -ENDobjQueryInterface(rule) - - -/* Exit the rule class. - * rgerhards, 2009-04-06 - */ -BEGINObjClassExit(rule, OBJ_IS_CORE_MODULE) /* class, version */ - objRelease(errmsg, CORE_COMPONENT); -ENDObjClassExit(rule) - - -/* Initialize the rule class. Must be called as the very first method - * before anything else is called inside this class. - * rgerhards, 2008-02-19 - */ -BEGINObjClassInit(rule, 1, OBJ_IS_CORE_MODULE) /* class, version */ - /* request objects we use */ - CHKiRet(objUse(errmsg, CORE_COMPONENT)); - - /* set our own handlers */ - OBJSetMethodHandler(objMethod_DEBUGPRINT, ruleDebugPrint); - OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, ruleConstructFinalize); -ENDObjClassInit(rule) - -/* vi:set ai: - */ diff --git a/runtime/rule.h b/runtime/rule.h deleted file mode 100644 index 1b07279b..00000000 --- a/runtime/rule.h +++ /dev/null @@ -1,78 +0,0 @@ -/* The rule object. - * - * This implements rules within rsyslog. - * - * Copyright 2009-2012 Adiscon GmbH. - * - * This file is part of the rsyslog runtime library. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * -or- - * see COPYING.ASL20 in the source distribution - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef INCLUDED_RULE_H -#define INCLUDED_RULE_H - -#include "libestr.h" -#include "linkedlist.h" -#include "regexp.h" -#include "rainerscript.h" - -/* the rule object */ -struct rule_s { - BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - /* filter properties */ - enum { - FILTER_PRI = 0, /* traditional PRI based filer */ - FILTER_PROP = 1, /* extended filter, property based */ - FILTER_EXPR = 2 /* extended filter, expression based */ - } f_filter_type; - EHostnameCmpMode eHostnameCmpMode; - cstr_t *pCSHostnameComp; /* hostname to check */ - cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ - union { - u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ - struct { - fiop_t operation; - regex_t *regex_cache; /* cache for compiled REs, if such are used */ - cstr_t *pCSCompValue; /* value to "compare" against */ - sbool isNegated; - propid_t propID; /* ID of the requested property */ - es_str_t *propName; /* name of property for CEE-based filters */ - } prop; - struct cnfexpr *expr; /* expression object */ - } f_filterData; - - ruleset_t *pRuleset; /* associated ruleset */ - linkedList_t llActList; /* list of configured actions */ -}; - -/* interfaces */ -BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ - INTERFACEObjDebugPrint(rule); - rsRetVal (*Construct)(rule_t **ppThis); - rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis); - rsRetVal (*Destruct)(rule_t **ppThis); - rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); - rsRetVal (*ProcessBatch)(rule_t *pThis, batch_t *pBatch); - rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*); - ruleset_t* (*GetAssRuleset)(rule_t *pThis); -ENDinterface(rule) -#define ruleCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ -/* change for v2: ProcessMsg replaced by ProcessBatch - 2010-06-10 */ - - -/* prototypes */ -PROTOTYPEObj(rule); - -#endif /* #ifndef INCLUDED_RULE_H */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 5cb34148..5bf7ac03 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -11,27 +11,24 @@ * * Module begun 2009-06-10 by Rainer Gerhards * - * Copyright 2009-2011 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * - * The rsyslog runtime library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The rsyslog runtime library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - #include "config.h" #include <stdlib.h> #include <assert.h> @@ -42,22 +39,36 @@ #include "cfsysline.h" #include "msg.h" #include "ruleset.h" -#include "rule.h" #include "errmsg.h" #include "parser.h" #include "batch.h" #include "unicode-helper.h" #include "rsconf.h" +#include "action.h" +#include "rainerscript.h" +#include "srUtils.h" +#include "modules.h" #include "dirty.h" /* for main ruleset queue creation */ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) -DEFobjCurrIf(rule) DEFobjCurrIf(parser) +/* tables for interfacing with the v6 config system (as far as we need to) */ +static struct cnfparamdescr rspdescr[] = { + { "name", eCmdHdlrString, CNFPARAM_REQUIRED }, + { "parser", eCmdHdlrArray, 0 } +}; +static struct cnfparamblk rspblk = + { CNFPARAMBLK_VERSION, + sizeof(rspdescr)/sizeof(struct cnfparamdescr), + rspdescr + }; + /* forward definitions */ static rsRetVal processBatch(batch_t *pBatch); +static rsRetVal scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active); /* ---------- linked-list key handling functions (ruleset) ---------- */ @@ -73,45 +84,61 @@ rulesetKeyDestruct(void __attribute__((unused)) *pData) /* ---------- END linked-list key handling functions (ruleset) ---------- */ +/* iterate over all actions in a script (stmt subtree) */ +static void +scriptIterateAllActions(struct cnfstmt *root, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + struct cnfstmt *stmt; + for(stmt = root ; stmt != NULL ; stmt = stmt->next) { + switch(stmt->nodetype) { + case S_NOP: + case S_STOP: + case S_CALL:/* call does not need to do anything - done in called ruleset! */ + break; + case S_ACT: + DBGPRINTF("iterateAllActions calling into action %p\n", stmt->d.act); + pFunc(stmt->d.act, pParam); + break; + case S_IF: + if(stmt->d.s_if.t_then != NULL) + scriptIterateAllActions(stmt->d.s_if.t_then, + pFunc, pParam); + if(stmt->d.s_if.t_else != NULL) + scriptIterateAllActions(stmt->d.s_if.t_else, + pFunc, pParam); + break; + case S_PRIFILT: + if(stmt->d.s_prifilt.t_then != NULL) + scriptIterateAllActions(stmt->d.s_prifilt.t_then, + pFunc, pParam); + if(stmt->d.s_prifilt.t_else != NULL) + scriptIterateAllActions(stmt->d.s_prifilt.t_else, + pFunc, pParam); + break; + case S_PROPFILT: + scriptIterateAllActions(stmt->d.s_propfilt.t_then, + pFunc, pParam); + break; + default: + dbgprintf("error: unknown stmt type %u during iterateAll\n", + (unsigned) stmt->nodetype); + break; + } + } +} /* driver to iterate over all of this ruleset actions */ typedef struct iterateAllActions_s { rsRetVal (*pFunc)(void*, void*); void *pParam; } iterateAllActions_t; -DEFFUNC_llExecFunc(doIterateRulesetActions) -{ - DEFiRet; - rule_t* pRule = (rule_t*) pData; - iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; - iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); - RETiRet; -} -/* iterate over all actions of THIS rule set. - */ -static rsRetVal -iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) -{ - iterateAllActions_t params; - DEFiRet; - assert(pFunc != NULL); - - params.pFunc = pFunc; - params.pParam = pParam; - CHKiRet(llExecFunc(&(pThis->llRules), doIterateRulesetActions, ¶ms)); - -finalize_it: - RETiRet; -} - - /* driver to iterate over all actions */ DEFFUNC_llExecFunc(doIterateAllActions) { DEFiRet; ruleset_t* pThis = (ruleset_t*) pData; iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; - iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); + scriptIterateAllActions(pThis->root, pMyParam->pFunc, pMyParam->pParam); RETiRet; } /* iterate over ALL actions present in the WHOLE system. @@ -134,30 +161,10 @@ finalize_it: } - -/* helper to processBatch(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(processBatchDoRules) -{ - rsRetVal iRet; - ISOBJ_TYPE_assert(pData, rule); - DBGPRINTF("Processing next rule\n"); - iRet = rule.ProcessBatch((rule_t*) pData, (batch_t*) pParam); - DBGPRINTF("ruleset: get iRet %d from rule.ProcessMsg()\n", iRet); - return iRet; -} - - - /* This function is similar to processBatch(), but works on a batch that * contains rules from multiple rulesets. In this case, we can not push * the whole batch through the ruleset. Instead, we examine it and * partition it into sub-rulesets which we then push through the system. - * Note that when we evaluate which message must be processed, we do NOT need - * to look at bFilterOK, because this value is only set in a later processing - * stage. Doing so caused a bug during development ;) * rgerhards, 2010-06-15 */ static inline rsRetVal @@ -174,7 +181,7 @@ processBatchMultiRuleset(batch_t *pBatch) do { bHaveUnprocessed = 0; /* search for first unprocessed element */ - for(iStart = 0 ; iStart < pBatch->nElem && pBatch->pElem[iStart].state == BATCH_STATE_DISC ; ++iStart) + for(iStart = 0 ; iStart < pBatch->nElem && pBatch->eltState[iStart] == BATCH_STATE_DISC ; ++iStart) /* just search, no action */; if(iStart == pBatch->nElem) break; /* everything processed */ @@ -187,11 +194,11 @@ processBatchMultiRuleset(batch_t *pBatch) for(i = iStart ; i < pBatch->nElem ; ++i) { if(batchElemGetRuleset(pBatch, i) == currRuleset) { /* for performance reasons, we copy only those members that we actually need */ - snglRuleBatch.pElem[iNew].pUsrp = pBatch->pElem[i].pUsrp; - snglRuleBatch.pElem[iNew].state = pBatch->pElem[i].state; + snglRuleBatch.pElem[iNew].pMsg = pBatch->pElem[i].pMsg; + snglRuleBatch.eltState[iNew] = pBatch->eltState[i]; ++iNew; /* We indicate the element also as done, so it will not be processed again */ - pBatch->pElem[i].state = BATCH_STATE_DISC; + pBatch->eltState[i] = BATCH_STATE_DISC; } else { bHaveUnprocessed = 1; } @@ -207,6 +214,348 @@ finalize_it: RETiRet; } +/* return a new "active" structure for the batch. Free with freeActive(). */ +static inline sbool *newActive(batch_t *pBatch) +{ + return malloc(sizeof(sbool) * batchNumMsgs(pBatch)); + +} +static inline void freeActive(sbool *active) { free(active); } + + +/* for details, see scriptExec() header comment! */ +/* call action for all messages with filter on */ +static rsRetVal +execAct(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + DEFiRet; +dbgprintf("RRRR: execAct [%s]: batch of %d elements, active %p\n", modGetName(stmt->d.act->pMod), batchNumMsgs(pBatch), active); + pBatch->active = active; + stmt->d.act->submitToActQ(stmt->d.act, pBatch); + RETiRet; +} + +static rsRetVal +execSet(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + int i; + struct var result; + DEFiRet; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if( pBatch->eltState[i] != BATCH_STATE_DISC + && (active == NULL || active[i])) { + cnfexprEval(stmt->d.s_set.expr, &result, pBatch->pElem[i].pMsg); + msgSetJSONFromVar(pBatch->pElem[i].pMsg, stmt->d.s_set.varname, + &result); + varDelete(&result); + } + } + RETiRet; +} + +static rsRetVal +execUnset(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + int i; + DEFiRet; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if( pBatch->eltState[i] != BATCH_STATE_DISC + && (active == NULL || active[i])) { + msgUnsetJSON(pBatch->pElem[i].pMsg, stmt->d.s_unset.varname); + } + } + RETiRet; +} + +/* for details, see scriptExec() header comment! */ +/* "stop" simply discards the filtered items - it's just a (hopefully more intuitive + * shortcut for users. + */ +static rsRetVal +execStop(batch_t *pBatch, sbool *active) +{ + int i; + DEFiRet; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if( pBatch->eltState[i] != BATCH_STATE_DISC + && (active == NULL || active[i])) { + pBatch->eltState[i] = BATCH_STATE_DISC; + } + } + RETiRet; +} + +/* for details, see scriptExec() header comment! */ +// save current filter, evaluate new one +// perform then (if any message) +// if ELSE given: +// set new filter, inverted +// perform else (if any messages) +static rsRetVal +execIf(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + sbool *newAct; + int i; + sbool bRet; + sbool allInactive = 1; + DEFiRet; + newAct = newActive(pBatch); + for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { + if(*(pBatch->pbShutdownImmediate)) + FINALIZE; + if(pBatch->eltState[i] == BATCH_STATE_DISC) + continue; /* will be ignored in any case */ + if(active == NULL || active[i]) { + bRet = cnfexprEvalBool(stmt->d.s_if.expr, pBatch->pElem[i].pMsg); + allInactive = 0; + } else + bRet = 0; + newAct[i] = bRet; + DBGPRINTF("batch: item %d: expr eval: %d\n", i, bRet); + } + + if(allInactive) { + DBGPRINTF("execIf: all batch elements are inactive, holding execution\n"); + freeActive(newAct); + FINALIZE; + } + + if(stmt->d.s_if.t_then != NULL) { + scriptExec(stmt->d.s_if.t_then, pBatch, newAct); + } + if(stmt->d.s_if.t_else != NULL) { + for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { + if(*(pBatch->pbShutdownImmediate)) + FINALIZE; + if(pBatch->eltState[i] != BATCH_STATE_DISC + && (active == NULL || active[i])) + newAct[i] = !newAct[i]; + } + scriptExec(stmt->d.s_if.t_else, pBatch, newAct); + } + freeActive(newAct); +finalize_it: + RETiRet; +} + +/* for details, see scriptExec() header comment! */ +static void +execPRIFILT(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + sbool *newAct; + msg_t *pMsg; + int bRet; + int i; + newAct = newActive(pBatch); + for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { + if(*(pBatch->pbShutdownImmediate)) + return; + if(pBatch->eltState[i] == BATCH_STATE_DISC) + continue; /* will be ignored in any case */ + pMsg = pBatch->pElem[i].pMsg; + if(active == NULL || active[i]) { + if( (stmt->d.s_prifilt.pmask[pMsg->iFacility] == TABLE_NOPRI) || + ((stmt->d.s_prifilt.pmask[pMsg->iFacility] + & (1<<pMsg->iSeverity)) == 0) ) + bRet = 0; + else + bRet = 1; + } else + bRet = 0; + newAct[i] = bRet; + DBGPRINTF("batch: item %d PRIFILT %d\n", i, newAct[i]); + } + + if(stmt->d.s_prifilt.t_then != NULL) { + scriptExec(stmt->d.s_prifilt.t_then, pBatch, newAct); + } + if(stmt->d.s_prifilt.t_else != NULL) { + for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { + if(*(pBatch->pbShutdownImmediate)) + return; + if(pBatch->eltState[i] != BATCH_STATE_DISC + && (active == NULL || active[i])) + newAct[i] = !newAct[i]; + } + scriptExec(stmt->d.s_prifilt.t_else, pBatch, newAct); + } + freeActive(newAct); +} + + +/* helper to execPROPFILT(), as the evaluation itself is quite lengthy */ +static int +evalPROPFILT(struct cnfstmt *stmt, msg_t *pMsg) +{ + unsigned short pbMustBeFreed; + uchar *pszPropVal; + int bRet = 0; + rs_size_t propLen; + + if(stmt->d.s_propfilt.propID == PROP_INVALID) + goto done; + + pszPropVal = MsgGetProp(pMsg, NULL, stmt->d.s_propfilt.propID, + stmt->d.s_propfilt.propName, &propLen, + &pbMustBeFreed, NULL); + + /* Now do the compares (short list currently ;)) */ + switch(stmt->d.s_propfilt.operation ) { + case FIOP_CONTAINS: + if(rsCStrLocateInSzStr(stmt->d.s_propfilt.pCSCompValue, (uchar*) pszPropVal) != -1) + bRet = 1; + break; + case FIOP_ISEMPTY: + if(propLen == 0) + bRet = 1; /* process message! */ + break; + case FIOP_ISEQUAL: + if(rsCStrSzStrCmp(stmt->d.s_propfilt.pCSCompValue, + pszPropVal, propLen) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_STARTSWITH: + if(rsCStrSzStrStartsWithCStr(stmt->d.s_propfilt.pCSCompValue, + pszPropVal, propLen) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_REGEX: + if(rsCStrSzStrMatchRegex(stmt->d.s_propfilt.pCSCompValue, + (unsigned char*) pszPropVal, 0, &stmt->d.s_propfilt.regex_cache) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(stmt->d.s_propfilt.pCSCompValue, + (unsigned char*) pszPropVal, 1, &stmt->d.s_propfilt.regex_cache) == RS_RET_OK) + bRet = 1; + break; + default: + /* here, it handles NOP (for performance reasons) */ + assert(stmt->d.s_propfilt.operation == FIOP_NOP); + bRet = 1; /* as good as any other default ;) */ + break; + } + + /* now check if the value must be negated */ + if(stmt->d.s_propfilt.isNegated) + bRet = (bRet == 1) ? 0 : 1; + + if(Debug) { + char *cstr; + if(stmt->d.s_propfilt.propID == PROP_CEE) { + cstr = es_str2cstr(stmt->d.s_propfilt.propName, NULL); + DBGPRINTF("Filter: check for CEE property '%s' (value '%s') ", + cstr, pszPropVal); + free(cstr); + } else { + DBGPRINTF("Filter: check for property '%s' (value '%s') ", + propIDToName(stmt->d.s_propfilt.propID), pszPropVal); + } + if(stmt->d.s_propfilt.isNegated) + DBGPRINTF("NOT "); + if(stmt->d.s_propfilt.operation == FIOP_ISEMPTY) { + DBGPRINTF("%s : %s\n", + getFIOPName(stmt->d.s_propfilt.operation), + bRet ? "TRUE" : "FALSE"); + } else { + DBGPRINTF("%s '%s': %s\n", + getFIOPName(stmt->d.s_propfilt.operation), + rsCStrGetSzStrNoNULL(stmt->d.s_propfilt.pCSCompValue), + bRet ? "TRUE" : "FALSE"); + } + } + + /* cleanup */ + if(pbMustBeFreed) + free(pszPropVal); +done: + return bRet; +} + +/* for details, see scriptExec() header comment! */ +static void +execPROPFILT(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + sbool *thenAct; + sbool bRet; + int i; + thenAct = newActive(pBatch); + for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { + if(*(pBatch->pbShutdownImmediate)) + return; + if(pBatch->eltState[i] == BATCH_STATE_DISC) + continue; /* will be ignored in any case */ + if(active == NULL || active[i]) { + bRet = evalPROPFILT(stmt, pBatch->pElem[i].pMsg); + } else + bRet = 0; + thenAct[i] = bRet; + DBGPRINTF("batch: item %d PROPFILT %d\n", i, thenAct[i]); + } + + scriptExec(stmt->d.s_propfilt.t_then, pBatch, thenAct); + freeActive(thenAct); +} + +/* The rainerscript execution engine. It is debatable if that would be better + * contained in grammer/rainerscript.c, HOWEVER, that file focusses primarily + * on the parsing and object creation part. So as an actual executor, it is + * better suited here. + * param active: if NULL, all messages are active (to be processed), if non-null + * this is an array of the same size as the batch. If 1, the message + * is to be processed, otherwise not. + * NOTE: this function must receive batches which contain a single ruleset ONLY! + * rgerhards, 2012-09-04 + */ +static rsRetVal +scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active) +{ + DEFiRet; + struct cnfstmt *stmt; + + for(stmt = root ; stmt != NULL ; stmt = stmt->next) { + if(Debug) { + dbgprintf("scriptExec: batch of %d elements, active %p, active[0]:%d\n", + batchNumMsgs(pBatch), active, (active == NULL ? 1 : active[0])); + cnfstmtPrintOnly(stmt, 2, 0); + } + switch(stmt->nodetype) { + case S_NOP: + break; + case S_STOP: + execStop(pBatch, active); + break; + case S_ACT: + execAct(stmt, pBatch, active); + break; + case S_SET: + execSet(stmt, pBatch, active); + break; + case S_UNSET: + execUnset(stmt, pBatch, active); + break; + case S_CALL: + scriptExec(stmt->d.s_call.stmt, pBatch, active); + break; + case S_IF: + execIf(stmt, pBatch, active); + break; + case S_PRIFILT: + execPRIFILT(stmt, pBatch, active); + break; + case S_PROPFILT: + execPROPFILT(stmt, pBatch, active); + break; + default: + dbgprintf("error: unknown stmt type %u during exec\n", + (unsigned) stmt->nodetype); + break; + } + } + RETiRet; +} + + /* Process (consume) a batch of messages. Calls the actions configured. * If the whole batch uses a singel ruleset, we can process the batch as * a whole. Otherwise, we need to process it slower, on a message-by-message @@ -226,7 +575,7 @@ processBatch(batch_t *pBatch) if(pThis == NULL) pThis = ourConf->rulesets.pDflt; ISOBJ_TYPE_assert(pThis, ruleset); - CHKiRet(llExecFunc(&pThis->llRules, processBatchDoRules, pBatch)); + CHKiRet(scriptExec(pThis->root, pBatch, NULL)); } else { CHKiRet(processBatchMultiRuleset(pBatch)); } @@ -248,34 +597,21 @@ GetParserList(rsconf_t *conf, msg_t *pMsg) } -/* Add a new rule to the end of the current rule set. We do a number - * of checks and ignore the rule if it does not pass them. - */ -static rsRetVal -addRule(ruleset_t *pThis, rule_t **ppRule) +/* Add a script block to the current ruleset */ +static void +addScript(ruleset_t *pThis, struct cnfstmt *script) { - int iActionCnt; - DEFiRet; - - ISOBJ_TYPE_assert(pThis, ruleset); - ISOBJ_TYPE_assert(*ppRule, rule); - - CHKiRet(llGetNumElts(&(*ppRule)->llActList, &iActionCnt)); - if(iActionCnt == 0) { - errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); - rule.Destruct(ppRule); - } else { - CHKiRet(llAppend(&pThis->llRules, NULL, *ppRule)); - DBGPRINTF("selector line successfully processed, %d actions\n", iActionCnt); + if(pThis->last == NULL) + pThis->root = pThis->last = script; + else { + pThis->last->next = script; + pThis->last = script; } - -finalize_it: - RETiRet; } /* set name for ruleset */ -static rsRetVal setName(ruleset_t *pThis, uchar *pszName) +static rsRetVal rulesetSetName(ruleset_t *pThis, uchar *pszName) { DEFiRet; free(pThis->pszName); @@ -344,8 +680,7 @@ finalize_it: } -/* Set a new current rule set. If the ruleset can not be found, no change happens. - */ +/* Set a new current rule set. If the ruleset can not be found, no change happens */ static rsRetVal SetCurrRuleset(rsconf_t *conf, uchar *pszName) { @@ -362,23 +697,11 @@ finalize_it: } -/* destructor we need to destruct rules inside our linked list contents. - */ -static rsRetVal -doRuleDestruct(void *pData) -{ - rule_t *pRule = (rule_t *) pData; - DEFiRet; - rule.Destruct(&pRule); - RETiRet; -} - - /* Standard-Constructor */ BEGINobjConstruct(ruleset) /* be sure to specify the object type also in END macro! */ - CHKiRet(llInit(&pThis->llRules, doRuleDestruct, NULL, NULL)); -finalize_it: + pThis->root = NULL; + pThis->last = NULL; ENDobjConstruct(ruleset) @@ -399,9 +722,6 @@ rulesetConstructFinalize(rsconf_t *conf, ruleset_t *pThis) CHKmalloc(keyName = ustrdup(pThis->pszName)); CHKiRet(llAppend(&(conf->rulesets.llRulesets), keyName, pThis)); - /* this now also is the new current ruleset */ - conf->rulesets.pCurr = pThis; - /* and also the default, if so far none has been set */ if(conf->rulesets.pDflt == NULL) conf->rulesets.pDflt = pThis; @@ -421,8 +741,8 @@ CODESTARTobjDestruct(ruleset) if(pThis->pParserLst != NULL) { parser.DestructParserList(&pThis->pParserLst); } - llDestroy(&pThis->llRules); free(pThis->pszName); + cnfstmtDestructLst(pThis->root); ENDobjDestruct(ruleset) @@ -456,16 +776,13 @@ rulesetDestructForLinkedList(void *pData) return rulesetDestruct(&pThis); } -/* helper for debugPrint(), initiates rule printing */ -DEFFUNC_llExecFunc(doDebugPrintRule) -{ - return rule.DebugPrint((rule_t*) pData); -} /* debugprint for the ruleset object */ BEGINobjDebugPrint(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDebugPrint(ruleset) dbgoprint((obj_t*) pThis, "rsyslog ruleset %s:\n", pThis->pszName); - llExecFunc(&pThis->llRules, doDebugPrintRule, NULL); + cnfstmtPrint(pThis->root, 0); + dbgoprint((obj_t*) pThis, "ruleset %s assigned parser list:\n", pThis->pszName); + printParserList(pThis->pParserLst); ENDobjDebugPrint(ruleset) @@ -486,6 +803,40 @@ debugPrintAll(rsconf_t *conf) RETiRet; } +static inline void +rulesetOptimize(ruleset_t *pRuleset) +{ + if(Debug) { + dbgprintf("ruleset '%s' before optimization:\n", + pRuleset->pszName); + rulesetDebugPrint((ruleset_t*) pRuleset); + } + cnfstmtOptimize(pRuleset->root); + if(Debug) { + dbgprintf("ruleset '%s' after optimization:\n", + pRuleset->pszName); + rulesetDebugPrint((ruleset_t*) pRuleset); + } +} + +/* helper for rulsetOptimizeAll(), optimizes a single ruleset */ +DEFFUNC_llExecFunc(doRulesetOptimizeAll) +{ + rulesetOptimize((ruleset_t*) pData); + return RS_RET_OK; +} +/* optimize all rulesets + */ +rsRetVal +rulesetOptimizeAll(rsconf_t *conf) +{ + DEFiRet; + dbgprintf("begin ruleset optimization phase\n"); + llExecFunc(&(conf->rulesets.llRulesets), doRulesetOptimizeAll, NULL); + dbgprintf("ruleset optimization phase finished.\n"); + RETiRet; +} + /* Create a ruleset-specific "main" queue for this ruleset. If one is already * defined, an error message is emitted but nothing else is done. @@ -517,7 +868,7 @@ doRulesetCreateQueue(rsconf_t *conf, int *pNewVal) rsname = (conf->rulesets.pCurr->pszName == NULL) ? (uchar*) "[ruleset]" : conf->rulesets.pCurr->pszName; DBGPRINTF("adding a ruleset-specific \"main\" queue for ruleset '%s'\n", rsname); - CHKiRet(createMainQueue(&conf->rulesets.pCurr->pQueue, rsname)); + CHKiRet(createMainQueue(&conf->rulesets.pCurr->pQueue, rsname, NULL)); finalize_it: RETiRet; @@ -539,13 +890,11 @@ rulesetCreateQueue(void __attribute__((unused)) *pVal, int *pNewVal) * rgerhards, 2009-11-04 */ static rsRetVal -doRulesetAddParser(rsconf_t *conf, uchar *pName) +doRulesetAddParser(ruleset_t *pRuleset, uchar *pName) { parser_t *pParser; DEFiRet; - assert(conf->rulesets.pCurr != NULL); - CHKiRet(objUse(parser, CORE_COMPONENT)); iRet = parser.FindParser(&pParser, pName); if(iRet == RS_RET_PARSER_NOT_FOUND) { @@ -557,9 +906,9 @@ doRulesetAddParser(rsconf_t *conf, uchar *pName) FINALIZE; } - CHKiRet(parser.AddParserToList(&conf->rulesets.pCurr->pParserLst, pParser)); + CHKiRet(parser.AddParserToList(&pRuleset->pParserLst, pParser)); - DBGPRINTF("added parser '%s' to ruleset '%s'\n", pName, conf->rulesets.pCurr->pszName); + DBGPRINTF("added parser '%s' to ruleset '%s'\n", pName, pRuleset->pszName); finalize_it: d_free(pName); /* no longer needed */ @@ -570,7 +919,72 @@ finalize_it: static rsRetVal rulesetAddParser(void __attribute__((unused)) *pVal, uchar *pName) { - return doRulesetAddParser(ourConf, pName); + return doRulesetAddParser(ourConf->rulesets.pCurr, pName); +} + + +/* Process ruleset() objects */ +rsRetVal +rulesetProcessCnf(struct cnfobj *o) +{ + struct cnfparamvals *pvals; + struct cnfparamvals *queueParams; + rsRetVal localRet; + uchar *rsName = NULL; + uchar *parserName; + int nameIdx, parserIdx; + ruleset_t *pRuleset; + struct cnfarray *ar; + int i; + uchar *rsname; + DEFiRet; + + pvals = nvlstGetParams(o->nvlst, &rspblk, NULL); + if(pvals == NULL) { + ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + } + DBGPRINTF("ruleset param blk after rulesetProcessCnf:\n"); + cnfparamsPrint(&rspblk, pvals); + nameIdx = cnfparamGetIdx(&rspblk, "name"); + rsName = (uchar*)es_str2cstr(pvals[nameIdx].val.d.estr, NULL); + localRet = rulesetGetRuleset(loadConf, &pRuleset, rsName); + if(localRet == RS_RET_OK) { + errmsg.LogError(0, RS_RET_RULESET_EXISTS, + "error: ruleset '%s' specified more than once", + rsName); + cnfstmtDestructLst(o->script); + ABORT_FINALIZE(RS_RET_RULESET_EXISTS); + } else if(localRet != RS_RET_NOT_FOUND) { + ABORT_FINALIZE(localRet); + } + CHKiRet(rulesetConstruct(&pRuleset)); + CHKiRet(rulesetSetName(pRuleset, rsName)); + CHKiRet(rulesetConstructFinalize(loadConf, pRuleset)); + addScript(pRuleset, o->script); + + /* we have only two params, so we do NOT do the usual param loop */ + parserIdx = cnfparamGetIdx(&rspblk, "parser"); + if(parserIdx != -1 && pvals[parserIdx].bUsed) { + ar = pvals[parserIdx].val.d.ar; + for(i = 0 ; i < ar->nmemb ; ++i) { + parserName = (uchar*)es_str2cstr(ar->arr[i], NULL); + doRulesetAddParser(pRuleset, parserName); + free(parserName); + } + } + + /* pick up ruleset queue parameters */ + qqueueDoCnfParams(o->nvlst, &queueParams); + if(queueCnfParamsSet(queueParams)) { + rsname = (pRuleset->pszName == NULL) ? (uchar*) "[ruleset]" : pRuleset->pszName; + DBGPRINTF("adding a ruleset-specific \"main\" queue for ruleset '%s'\n", rsname); + CHKiRet(createMainQueue(&pRuleset->pQueue, rsname, queueParams)); + } + +finalize_it: + free(rsName); + cnfparamvalsDestruct(pvals, &rspblk); + RETiRet; } @@ -595,9 +1009,9 @@ CODESTARTobjQueryInterface(ruleset) pIf->IterateAllActions = iterateAllActions; pIf->DestructAllActions = destructAllActions; - pIf->AddRule = addRule; + pIf->AddScript = addScript; pIf->ProcessBatch = processBatch; - pIf->SetName = setName; + pIf->SetName = rulesetSetName; pIf->DebugPrintAll = debugPrintAll; pIf->GetCurrent = GetCurrent; pIf->GetRuleset = rulesetGetRuleset; @@ -614,7 +1028,6 @@ ENDobjQueryInterface(ruleset) */ BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ objRelease(errmsg, CORE_COMPONENT); - objRelease(rule, CORE_COMPONENT); objRelease(parser, CORE_COMPONENT); ENDObjClassExit(ruleset) @@ -626,7 +1039,6 @@ ENDObjClassExit(ruleset) BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); - CHKiRet(objUse(rule, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, rulesetDebugPrint); diff --git a/runtime/ruleset.h b/runtime/ruleset.h index f4443e18..cbf8243b 100644 --- a/runtime/ruleset.h +++ b/runtime/ruleset.h @@ -25,13 +25,15 @@ #include "queue.h" #include "linkedlist.h" +#include "rsconf.h" /* the ruleset object */ struct ruleset_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ uchar *pszName; /* name of our ruleset */ qqueue_t *pQueue; /* "main" message queue, if the ruleset has its own (else NULL) */ + struct cnfstmt *root; + struct cnfstmt *last; parserList_t *pParserLst;/* list of parsers to use for this ruleset */ }; @@ -42,9 +44,7 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Construct)(ruleset_t **ppThis); rsRetVal (*ConstructFinalize)(rsconf_t *conf, ruleset_t __attribute__((unused)) *pThis); rsRetVal (*Destruct)(ruleset_t **ppThis); - rsRetVal (*IterateAllActions)(rsconf_t *conf, rsRetVal (*pFunc)(void*, void*), void* pParam); rsRetVal (*DestructAllActions)(rsconf_t *conf); - rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); rsRetVal (*ProcessBatch)(batch_t*); rsRetVal (*GetRuleset)(rsconf_t *conf, ruleset_t **ppThis, uchar*); @@ -60,8 +60,12 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ * removed conf ptr from SetName, AddRule as the flex/bison based * system uses globals in any case. */ + /* v7, 2012-09-04 */ + /* AddRule() removed */ + /*TODO:REMOVE*/rsRetVal (*IterateAllActions)(rsconf_t *conf, rsRetVal (*pFunc)(void*, void*), void* pParam); + void (*AddScript)(ruleset_t *pThis, struct cnfstmt *script); ENDinterface(ruleset) -#define rulesetCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ +#define rulesetCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* prototypes */ @@ -87,5 +91,16 @@ rulesetGetName(ruleset_t *pRuleset) } +/* we will most probably convert this module back to traditional C + * calling sequence, so here we go... + */ rsRetVal rulesetGetRuleset(rsconf_t *conf, ruleset_t **ppRuleset, uchar *pszName); +rsRetVal rulesetOptimizeAll(rsconf_t *conf); +rsRetVal rulesetProcessCnf(struct cnfobj *o); + +/* Set a current rule set to already-known pointer */ +static inline void +rulesetSetCurrRulesetPtr(ruleset_t *pRuleset) { + loadConf->rulesets.pCurr = pRuleset; +} #endif /* #ifndef INCLUDED_RULESET_H */ diff --git a/runtime/sigprov.h b/runtime/sigprov.h new file mode 100644 index 00000000..82587b7d --- /dev/null +++ b/runtime/sigprov.h @@ -0,0 +1,37 @@ +/* The interface definition for (file) signature providers. + * + * This is just an abstract driver interface, which needs to be + * implemented by concrete classes. + * + * Copyright 2013 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_SIGPROV_H +#define INCLUDED_SIGPROV_H + +/* interface */ +BEGINinterface(sigprov) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(void *ppThis); + rsRetVal (*SetCnfParam)(void *ppThis, struct nvlst *lst); + rsRetVal (*Destruct)(void *ppThis); + rsRetVal (*OnFileOpen)(void *pThis, uchar *fn, void *pFileInstData); + rsRetVal (*OnRecordWrite)(void *pFileInstData, uchar *rec, rs_size_t lenRec); + rsRetVal (*OnFileClose)(void *pFileInstData); +ENDinterface(sigprov) +#define sigprovCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#endif /* #ifndef INCLUDED_SIGPROV_H */ diff --git a/runtime/srUtils.h b/runtime/srUtils.h index 3169fd94..8626a4bb 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -91,6 +91,7 @@ char *rs_strerror_r(int errnum, char *buf, size_t buflen); int decodeSyslogName(uchar *name, syslogName_t *codetab); int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); rsRetVal getFileSize(uchar *pszName, off_t *pSize); +int containsGlobWildcard(char *str); /* mutex operations */ /* some useful constants */ diff --git a/runtime/srutils.c b/runtime/srutils.c index f420c0f7..6a509b4a 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -92,6 +92,9 @@ syslogName_t syslogFacNames[] = { #if defined(LOG_FTP) {"ftp", LOG_FTP}, #endif +#if defined(LOG_AUDIT) + {"audit", LOG_AUDIT}, +#endif {"local0", LOG_LOCAL0}, {"local1", LOG_LOCAL1}, {"local2", LOG_LOCAL2}, @@ -524,8 +527,7 @@ char *rs_strerror_r(int errnum, char *buf, size_t buflen) { } -/* Decode a symbolic name to a numeric value - */ +/* Decode a symbolic name to a numeric value */ int decodeSyslogName(uchar *name, syslogName_t *codetab) { register syslogName_t *c; @@ -535,22 +537,23 @@ int decodeSyslogName(uchar *name, syslogName_t *codetab) ASSERT(name != NULL); ASSERT(codetab != NULL); - dbgprintf("symbolic name: %s", name); - if (isdigit((int) *name)) - { - dbgprintf("\n"); + DBGPRINTF("symbolic name: %s", name); + if(isdigit((int) *name)) { + DBGPRINTF("\n"); return (atoi((char*) name)); } strncpy((char*) buf, (char*) name, 79); - for (p = buf; *p; p++) + for(p = buf; *p; p++) { if (isupper((int) *p)) *p = tolower((int) *p); - for (c = codetab; c->c_name; c++) - if (!strcmp((char*) buf, (char*) c->c_name)) - { - dbgprintf(" ==> %d\n", c->c_val); + } + for(c = codetab; c->c_name; c++) { + if(!strcmp((char*) buf, (char*) c->c_name)) { + DBGPRINTF(" ==> %d\n", c->c_val); return (c->c_val); } + } + DBGPRINTF("\n"); return (-1); } @@ -627,6 +630,28 @@ finalize_it: RETiRet; } +/* Returns 1 if the given string contains a non-escaped glob(3) + * wildcard character and 0 otherwise (or if the string is empty). + */ +int +containsGlobWildcard(char *str) +{ + char *p; + if(!str) { + return 0; + } + /* From Linux Programmer's Guide: + * "A string is a wildcard pattern if it contains one of the characters '?', '*' or '['" + * "One can remove the special meaning of '?', '*' and '[' by preceding them by a backslash" + */ + for(p = str; *p != '\0'; p++) { + if((*p == '?' || *p == '*' || *p == '[') && + (p == str || *(p-1) != '\\')) { + return 1; + } + } + return 0; +} /* vim:set ai: */ diff --git a/runtime/stream.c b/runtime/stream.c index 3eb7708d..94fc0ca7 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -16,7 +16,7 @@ * it turns out to be problematic. Then, we need to quasi-refcount the number of accesses * to the object. * - * Copyright 2008-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -45,6 +45,7 @@ #include <pthread.h> #include <fcntl.h> #include <unistd.h> +#include <sys/types.h> #include <sys/stat.h> /* required for HP UX */ #include <errno.h> #include <pthread.h> @@ -56,6 +57,7 @@ #include "stream.h" #include "unicode-helper.h" #include "module-template.h" +#include "cryprov.h" #if HAVE_SYS_PRCTL_H # include <sys/prctl.h> #endif @@ -65,7 +67,6 @@ # define O_LARGEFILE 0 #endif #ifndef HAVE_LSEEK64 - typedef off_t off64_t; # define lseek64(fd, offset, whence) lseek(fd, offset, whence) #endif @@ -74,12 +75,14 @@ DEFobjStaticHelpers DEFobjCurrIf(zlibw) /* forward definitions */ -static rsRetVal strmFlushInternal(strm_t *pThis); +static rsRetVal strmFlushInternal(strm_t *pThis, int bFlushZip); static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); static rsRetVal strmCloseFile(strm_t *pThis); static void *asyncWriterThread(void *pPtr); -static rsRetVal doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlush); +static rsRetVal doZipFinish(strm_t *pThis); static rsRetVal strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal strmSeekCurrOffs(strm_t *pThis); /* methods */ @@ -196,6 +199,7 @@ static rsRetVal doPhysOpen(strm_t *pThis) { int iFlags = 0; + struct stat statOpen; DEFiRet; ISOBJ_TYPE_assert(pThis, strm); @@ -233,15 +237,75 @@ doPhysOpen(strm_t *pThis) ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); else ABORT_FINALIZE(RS_RET_IO_ERROR); + } + + if(pThis->tOperationsMode == STREAMMODE_READ) { + if(fstat(pThis->fd, &statOpen) == -1) { + DBGPRINTF("Error: cannot obtain inode# for file %s\n", pThis->pszCurrFName); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } + pThis->inode = statOpen.st_ino; + } + + if(!ustrcmp(pThis->pszCurrFName, UCHAR_CONSTANT(_PATH_CONSOLE)) || isatty(pThis->fd)) { + DBGPRINTF("file %d is a tty-type file\n", pThis->fd); + pThis->bIsTTY = 1; + } else { + pThis->bIsTTY = 0; + } + + if(pThis->cryprov != NULL) { + CHKiRet(pThis->cryprov->OnFileOpen(pThis->cryprovData, + pThis->pszCurrFName, &pThis->cryprovFileData)); + } +finalize_it: + RETiRet; +} + + +static rsRetVal +strmSetCurrFName(strm_t *pThis) +{ + DEFiRet; + + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); } else { - if(!ustrcmp(pThis->pszCurrFName, UCHAR_CONSTANT(_PATH_CONSOLE)) || isatty(pThis->fd)) { - DBGPRINTF("file %d is a tty-type file\n", pThis->fd); - pThis->bIsTTY = 1; + if(pThis->pszDir == NULL) { + if((pThis->pszCurrFName = ustrdup(pThis->pszFName)) == NULL) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } else { - pThis->bIsTTY = 0; + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, -1, 0)); } } +finalize_it: + RETiRet; +} + +/* This function checks if the actual file has changed and, if so, resets the + * offset. This is support for monitoring files. It should be called after + * deserializing the strm object and before doing any other operation on it + * (most importantly not an open or seek!). + */ +static rsRetVal +CheckFileChange(strm_t *pThis) +{ + struct stat statName; + DEFiRet; + CHKiRet(strmSetCurrFName(pThis)); + if(stat((char*) pThis->pszCurrFName, &statName) == -1) + ABORT_FINALIZE(RS_RET_IO_ERROR); + DBGPRINTF("stream/after deserialize checking for file change on '%s', " + "inode %u/%u, size/currOffs %llu/%llu\n", + pThis->pszCurrFName, (unsigned) pThis->inode, + (unsigned) statName.st_ino, statName.st_size, pThis->iCurrOffs); + if(pThis->inode != statName.st_ino || statName.st_size < pThis->iCurrOffs) { + DBGPRINTF("stream: file %s has changed\n", pThis->pszCurrFName); + pThis->iCurrOffs = 0; + } finalize_it: RETiRet; } @@ -264,19 +328,8 @@ static rsRetVal strmOpenFile(strm_t *pThis) if(pThis->pszFName == NULL) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { - CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, - pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); - } else { - if(pThis->pszDir == NULL) { - if((pThis->pszCurrFName = ustrdup(pThis->pszFName)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } else { - CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, - pThis->pszFName, pThis->lenFName, -1, 0)); - } - } - + CHKiRet(strmSetCurrFName(pThis)); + CHKiRet(doPhysOpen(pThis)); pThis->iCurrOffs = 0; @@ -334,6 +387,7 @@ strmWaitAsyncWriterDone(strm_t *pThis) */ static rsRetVal strmCloseFile(strm_t *pThis) { + off64_t currOffs; DEFiRet; ASSERT(pThis != NULL); @@ -341,7 +395,10 @@ static rsRetVal strmCloseFile(strm_t *pThis) (pThis->pszFName == NULL) ? "N/A" : (char*)pThis->pszFName); if(pThis->tOperationsMode != STREAMMODE_READ) { - strmFlushInternal(pThis); + strmFlushInternal(pThis, 0); + if(pThis->iZipLevel) { + doZipFinish(pThis); + } if(pThis->bAsyncWrite) { strmWaitAsyncWriterDone(pThis); } @@ -351,8 +408,14 @@ static rsRetVal strmCloseFile(strm_t *pThis) * against this. -- rgerhards, 2010-03-19 */ if(pThis->fd != -1) { + currOffs = lseek64(pThis->fd, 0, SEEK_CUR); close(pThis->fd); pThis->fd = -1; + pThis->inode = 0; + if(pThis->cryprov != NULL) { + pThis->cryprov->OnFileClose(pThis->cryprovFileData, currOffs); + pThis->cryprovFileData = NULL; + } } if(pThis->fdDir != -1) { @@ -361,7 +424,13 @@ static rsRetVal strmCloseFile(strm_t *pThis) pThis->fdDir = -1; } - if(pThis->bDeleteOnClose && pThis->pszCurrFName != NULL) { + if(pThis->bDeleteOnClose) { + if(pThis->pszCurrFName == NULL) { + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, + pThis->iFileNumDigits)); + } + DBGPRINTF("strmCloseFile: deleting '%s'\n", pThis->pszCurrFName); if(unlink((char*) pThis->pszCurrFName) == -1) { char errStr[1024]; int err = errno; @@ -369,12 +438,13 @@ static rsRetVal strmCloseFile(strm_t *pThis) DBGPRINTF("error %d unlinking '%s' - ignored: %s\n", errno, pThis->pszCurrFName, errStr); } - free(pThis->pszCurrFName); /* no longer needed in any case (just for open) */ + free(pThis->pszCurrFName); pThis->pszCurrFName = NULL; } pThis->iCurrOffs = 0; /* we are back at begin of file */ +finalize_it: RETiRet; } @@ -421,18 +491,15 @@ static rsRetVal strmHandleEOFMonitor(strm_t *pThis) { DEFiRet; - struct stat statOpen; struct stat statName; ISOBJ_TYPE_assert(pThis, strm); - if(fstat(pThis->fd, &statOpen) == -1) - ABORT_FINALIZE(RS_RET_IO_ERROR); if(stat((char*) pThis->pszCurrFName, &statName) == -1) ABORT_FINALIZE(RS_RET_IO_ERROR); - DBGPRINTF("stream checking for file change on '%s', inode %u/%u", - pThis->pszCurrFName, (unsigned) statOpen.st_ino, + DBGPRINTF("stream checking for file change on '%s', inode %u/%u\n", + pThis->pszCurrFName, (unsigned) pThis->inode, (unsigned) statName.st_ino); - if(statOpen.st_ino == statName.st_ino) { + if(pThis->inode == statName.st_ino) { ABORT_FINALIZE(RS_RET_EOF); } else { /* we had a file change! */ @@ -585,25 +652,33 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) * mode = 2 LF <not whitespace> mode, a log line starts at the beginning of a line, but following lines that are indented are part of the same log entry * This modal interface is not nearly as flexible as being able to define a regex for when a new record starts, but it's also not nearly as hard (or as slow) to implement */ - DEFiRet; uchar c; uchar finished; + rsRetVal readCharRet; + DEFiRet; ASSERT(pThis != NULL); ASSERT(ppCStr != NULL); CHKiRet(cstrConstruct(ppCStr)); - - /* now read the line */ CHKiRet(strmReadChar(pThis, &c)); - if (mode == 0){ - while(c != '\n') { + + if(mode == 0) { + /* append previous message to current message if necessary */ + if(pThis->prevLineSegment != NULL) { + CHKiRet(cstrAppendCStr(*ppCStr, pThis->prevLineSegment)); + cstrDestruct(&pThis->prevLineSegment); + } + while(c != '\n') { CHKiRet(cstrAppendChar(*ppCStr, c)); - CHKiRet(strmReadChar(pThis, &c)); + readCharRet = strmReadChar(pThis, &c); + if(readCharRet == RS_RET_EOF) {/* end of file reached without \n? */ + CHKiRet(rsCStrConstructFromCStr(&pThis->prevLineSegment, *ppCStr)); + } + CHKiRet(readCharRet); } CHKiRet(cstrFinalize(*ppCStr)); - } - if (mode == 1){ + } else if(mode == 1) { finished=0; while(finished == 0){ if(c != '\n') { @@ -624,8 +699,7 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) } } CHKiRet(cstrFinalize(*ppCStr)); - } - if (mode == 2){ + } else if(mode == 2) { /* indented follow-up lines */ finished=0; while(finished == 0){ @@ -675,9 +749,11 @@ BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! pThis->fd = -1; pThis->fdDir = -1; pThis->iUngetC = -1; + pThis->bVeryReliableZip = 0; pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); pThis->tOpenMode = 0600; + pThis->prevLineSegment = NULL; ENDobjConstruct(strm) @@ -777,6 +853,7 @@ stopWriter(strm_t *pThis) BEGINobjDestruct(strm) /* be sure to specify the object type also in END and CODESTART macros! */ int i; CODESTARTobjDestruct(strm) + /* we need to stop the ZIP writer */ if(pThis->bAsyncWrite) /* Note: mutex will be unlocked in stopWriter! */ d_pthread_mutex_lock(&pThis->mut); @@ -919,14 +996,14 @@ finalize_it: /* write memory buffer to a stream object. */ static inline rsRetVal -doWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +doWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlush) { DEFiRet; ASSERT(pThis != NULL); if(pThis->iZipLevel) { - CHKiRet(doZipWrite(pThis, pBuf, lenBuf)); + CHKiRet(doZipWrite(pThis, pBuf, lenBuf, bFlush)); } else { /* write without zipping */ CHKiRet(strmPhysWrite(pThis, pBuf, lenBuf)); @@ -971,7 +1048,7 @@ doAsyncWriteInternal(strm_t *pThis, size_t lenBuf) * the background thread. -- rgerhards, 2009-07-07 */ static rsRetVal -strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlushZip) { DEFiRet; @@ -990,7 +1067,7 @@ strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) if(pThis->bAsyncWrite) { CHKiRet(doAsyncWriteInternal(pThis, lenBuf)); } else { - CHKiRet(doWriteInternal(pThis, pBuf, lenBuf)); + CHKiRet(doWriteInternal(pThis, pBuf, lenBuf, bFlushZip)); } @@ -1011,17 +1088,20 @@ asyncWriterThread(void *pPtr) sbool bTimedOut = 0; strm_t *pThis = (strm_t*) pPtr; int err; + uchar thrdName[256] = "rs:"; ISOBJ_TYPE_assert(pThis, strm); BEGINfunc + ustrncpy(thrdName+3, pThis->pszFName, sizeof(thrdName)-4); + dbgOutputTID((char*)thrdName); # if HAVE_PRCTL && defined PR_SET_NAME - if(prctl(PR_SET_NAME, "rs:asyn strmwr", 0, 0, 0) != 0) { + if(prctl(PR_SET_NAME, (char*)thrdName, 0, 0, 0) != 0) { DBGPRINTF("prctl failed, not setting thread name for '%s'\n", "stream writer"); } # endif + d_pthread_mutex_lock(&pThis->mut); while(1) { /* loop broken inside */ - d_pthread_mutex_lock(&pThis->mut); while(pThis->iCnt == 0) { if(pThis->bStopWriter) { pthread_cond_broadcast(&pThis->isEmpty); @@ -1030,18 +1110,17 @@ asyncWriterThread(void *pPtr) } if(bTimedOut && pThis->iBufPtr > 0) { /* if we timed out, we need to flush pending data */ - strmFlushInternal(pThis); + strmFlushInternal(pThis, 0); bTimedOut = 0; - continue; /* now we should have data */ + d_pthread_mutex_unlock(&pThis->mut); + continue; } bTimedOut = 0; timeoutComp(&t, pThis->iFlushInterval * 1000); /* *1000 millisconds */ if(pThis->bDoTimedWait) { if((err = pthread_cond_timedwait(&pThis->notEmpty, &pThis->mut, &t)) != 0) { - if(err == ETIMEDOUT) { - bTimedOut = 1; - } else { - bTimedOut = 1; + bTimedOut = 1; /* simulate in any case */ + if(err != ETIMEDOUT) { char errStr[1024]; rs_strerror_r(err, errStr, sizeof(errStr)); DBGPRINTF("stream async writer timeout with error (%d): %s - ignoring\n", @@ -1056,8 +1135,12 @@ asyncWriterThread(void *pPtr) bTimedOut = 0; /* we may have timed out, but there *is* work to do... */ iDeq = pThis->iDeq++ % STREAM_ASYNC_NUMBUFS; - doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf); + + /* now we can do the actual write in parallel */ + d_pthread_mutex_unlock(&pThis->mut); + doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf, 0); // TODO: flush state // TODO: error check????? 2009-07-06 + d_pthread_mutex_lock(&pThis->mut); --pThis->iCnt; if(pThis->iCnt < STREAM_ASYNC_NUMBUFS) { @@ -1065,8 +1148,8 @@ asyncWriterThread(void *pPtr) if(pThis->iCnt == 0) pthread_cond_broadcast(&pThis->isEmpty); } - d_pthread_mutex_unlock(&pThis->mut); } + d_pthread_mutex_unlock(&pThis->mut); finalize_it: ENDfunc @@ -1128,9 +1211,16 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) DEFiRet; ISOBJ_TYPE_assert(pThis, strm); + DBGPRINTF("strmPhysWrite, stream %p, len %u\n", pThis, (unsigned)lenBuf); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); + /* here we place our crypto interface */ + if(pThis->cryprov != NULL) { + pThis->cryprov->Encrypt(pThis->cryprovFileData, pBuf, &lenBuf); + } + /* end crypto */ + iWritten = lenBuf; CHKiRet(doWriteCall(pThis, pBuf, &iWritten)); @@ -1166,63 +1256,97 @@ finalize_it: * rgerhards, 2009-06-04 */ static rsRetVal -doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlush) { - z_stream zstrm; int zRet; /* zlib return state */ - sbool bzInitDone = RSFALSE; DEFiRet; + unsigned outavail; assert(pThis != NULL); assert(pBuf != NULL); - /* allocate deflate state */ - zstrm.zalloc = Z_NULL; - zstrm.zfree = Z_NULL; - zstrm.opaque = Z_NULL; - zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ - /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - if(zRet != Z_OK) { - DBGPRINTF("error %d returned from zlib/deflateInit2()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); + if(!pThis->bzInitDone) { + /* allocate deflate state */ + pThis->zstrm.zalloc = Z_NULL; + pThis->zstrm.zfree = Z_NULL; + pThis->zstrm.opaque = Z_NULL; + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&pThis->zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateInit2()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + pThis->bzInitDone = RSTRUE; } - bzInitDone = RSTRUE; /* now doing the compression */ - zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ - zstrm.avail_in = lenBuf; + pThis->zstrm.next_in = (Bytef*) pBuf; + pThis->zstrm.avail_in = lenBuf; /* run deflate() on buffer until everything has been compressed */ do { - DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); - zstrm.avail_out = pThis->sIOBufSize; - zstrm.next_out = pThis->pZipBuf; - zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ - DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); - assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - if(zstrm.avail_out == pThis->sIOBufSize) - break; /* this is valid, indicates end of compression --> see zlib howto */ - CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, pThis->sIOBufSize - zstrm.avail_out)); - } while (zstrm.avail_out == 0); - assert(zstrm.avail_in == 0); /* all input will be used */ + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pThis->zstrm.avail_in, pThis->zstrm.total_in); + pThis->zstrm.avail_out = pThis->sIOBufSize; + pThis->zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&pThis->zstrm, bFlush ? Z_SYNC_FLUSH : Z_NO_FLUSH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pThis->zstrm.avail_out); + outavail =pThis->sIOBufSize - pThis->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); + } + } while (pThis->zstrm.avail_out == 0); finalize_it: - if(bzInitDone) { - zRet = zlibw.DeflateEnd(&zstrm); - if(zRet != Z_OK) { - DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); - } + if(pThis->bzInitDone && pThis->bVeryReliableZip) { + doZipFinish(pThis); } - RETiRet; } + +/* finish zlib buffer, to be called before closing the ZIP file (if + * running in stream mode). + */ +static rsRetVal +doZipFinish(strm_t *pThis) +{ + int zRet; /* zlib return state */ + DEFiRet; + unsigned outavail; + assert(pThis != NULL); + + if(!pThis->bzInitDone) + goto done; + + pThis->zstrm.avail_in = 0; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pThis->zstrm.avail_in, pThis->zstrm.total_in); + pThis->zstrm.avail_out = pThis->sIOBufSize; + pThis->zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&pThis->zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pThis->zstrm.avail_out); + outavail = pThis->sIOBufSize - pThis->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); + } + } while (pThis->zstrm.avail_out == 0); + +finalize_it: + zRet = zlibw.DeflateEnd(&pThis->zstrm); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); + } + + pThis->bzInitDone = 0; +done: RETiRet; +} + /* flush stream output buffer to persistent storage. This can be called at any time * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 */ static rsRetVal -strmFlushInternal(strm_t *pThis) +strmFlushInternal(strm_t *pThis, int bFlushZip) { DEFiRet; @@ -1232,7 +1356,7 @@ strmFlushInternal(strm_t *pThis) (long) pThis->iBufPtr, (pThis->iBufPtr == 0) ? " (no need to flush)" : ""); if(pThis->tOperationsMode != STREAMMODE_READ && pThis->iBufPtr > 0) { - iRet = strmSchedWrite(pThis, pThis->pIOBuf, pThis->iBufPtr); + iRet = strmSchedWrite(pThis, pThis->pIOBuf, pThis->iBufPtr, bFlushZip); } RETiRet; @@ -1254,7 +1378,7 @@ strmFlush(strm_t *pThis) if(pThis->bAsyncWrite) d_pthread_mutex_lock(&pThis->mut); - CHKiRet(strmFlushInternal(pThis)); + CHKiRet(strmFlushInternal(pThis, 1)); finalize_it: if(pThis->bAsyncWrite) @@ -1277,11 +1401,15 @@ static rsRetVal strmSeek(strm_t *pThis, off64_t offs) if(pThis->fd == -1) { CHKiRet(strmOpenFile(pThis)); } else { - CHKiRet(strmFlushInternal(pThis)); + CHKiRet(strmFlushInternal(pThis, 0)); } long long i; DBGOPRINT((obj_t*) pThis, "file %d seek, pos %llu\n", pThis->fd, (long long unsigned) offs); - i = lseek64(pThis->fd, offs, SEEK_SET); // TODO: check error! + i = lseek64(pThis->fd, offs, SEEK_SET); + if(i != offs) { + DBGPRINTF("strmSeek: error %lld seeking to offset %lld\n", i, offs); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } pThis->iCurrOffs = offs; /* we are now at *this* offset */ pThis->iBufPtr = 0; /* buffer invalidated */ @@ -1289,6 +1417,56 @@ finalize_it: RETiRet; } +/* multi-file seek, seeks to file number & offset within file. This + * is a support function for the queue, in circular mode. DO NOT USE + * IT FOR OTHER NEEDS - it may not work as expected. It will + * seek to the new position and delete interim files, as it skips them. + * Note: this code can be removed when the queue gets a new disk store + * handler (if and when it does ;)). + * The output parameter bytesDel receives the number of bytes that have + * been deleted (if a file is deleted) or 0 if nothing was deleted. + * rgerhards, 2012-11-07 + */ +rsRetVal +strmMultiFileSeek(strm_t *pThis, int FNum, off64_t offs, off64_t *bytesDel) +{ + struct stat statBuf; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + + if(FNum == 0 && offs == 0) { /* happens during queue init */ + *bytesDel = 0; + FINALIZE; + } + + if(pThis->iCurrFNum != FNum) { + /* Note: we assume that no more than one file is skipped - an + * assumption that is being used also by the whole rest of the + * code and most notably the queue subsystem. + */ + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, + pThis->iFileNumDigits)); + stat((char*)pThis->pszCurrFName, &statBuf); + *bytesDel = statBuf.st_size; + DBGPRINTF("strmMultiFileSeek: detected new filenum, was %d, new %d, " + "deleting '%s' (%lld bytes)\n", pThis->iCurrFNum, FNum, + pThis->pszCurrFName, (long long) *bytesDel); + unlink((char*)pThis->pszCurrFName); + free(pThis->pszCurrFName); + pThis->pszCurrFName = NULL; + pThis->iCurrFNum = FNum; + } else { + *bytesDel = 0; + } + pThis->iCurrOffs = offs; + +finalize_it: + RETiRet; +} + + /* seek to current offset. This is primarily a helper to readjust the OS file * pointer after a strm object has been deserialized. @@ -1320,7 +1498,7 @@ static rsRetVal strmWriteChar(strm_t *pThis, uchar c) /* if the buffer is full, we need to flush before we can write */ if(pThis->iBufPtr == pThis->sIOBufSize) { - CHKiRet(strmFlushInternal(pThis)); + CHKiRet(strmFlushInternal(pThis, 0)); } /* we now always have space for one character, so we simply copy it */ *(pThis->pIOBuf + pThis->iBufPtr) = c; @@ -1380,17 +1558,17 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) ASSERT(pThis != NULL); ASSERT(pBuf != NULL); -//DBGPRINTF("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n", pThis, pBuf,lenBuf, pThis->bDisabled, pThis->iSizeLimit, pThis->iCurrOffs); - if(pThis->bAsyncWrite) - d_pthread_mutex_lock(&pThis->mut); - + /* DEV DEBUG ONLY DBGPRINTF("strmWrite(%p[%s], '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n", pThis, pThis->pszCurrFName, pBuf,(long) lenBuf, pThis->bDisabled, (long) pThis->iSizeLimit, (long long) pThis->iCurrOffs); */ if(pThis->bDisabled) ABORT_FINALIZE(RS_RET_STREAM_DISABLED); + if(pThis->bAsyncWrite) + d_pthread_mutex_lock(&pThis->mut); + iOffset = 0; do { if(pThis->iBufPtr == pThis->sIOBufSize) { - CHKiRet(strmFlushInternal(pThis)); /* get a new buffer for rest of data */ + CHKiRet(strmFlushInternal(pThis, 0)); /* get a new buffer for rest of data */ } iWrite = pThis->sIOBufSize - pThis->iBufPtr; /* this fits in current buf */ if(iWrite > lenBuf) @@ -1405,7 +1583,7 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) * write it. This seems more natural than waiting (hours?) for the next message... */ if(pThis->iBufPtr == pThis->sIOBufSize) { - CHKiRet(strmFlushInternal(pThis)); /* get a new buffer for rest of data */ + CHKiRet(strmFlushInternal(pThis, 0)); /* get a new buffer for rest of data */ } finalize_it: @@ -1433,11 +1611,14 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, bVeryReliableZip, int) DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) DEFpropSetMeth(strm, iSizeLimit, off_t) DEFpropSetMeth(strm, iFlushInterval, int) DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) +DEFpropSetMeth(strm, cryprov, cryprov_if_t*) +DEFpropSetMeth(strm, cryprovData, void*) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { @@ -1564,7 +1745,7 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) ISOBJ_TYPE_assert(pThis, strm); ISOBJ_TYPE_assert(pStrm, strm); - strmFlushInternal(pThis); + strmFlushInternal(pThis, 0); CHKiRet(obj.BeginSerialize(pStrm, (obj_t*) pThis)); objSerializeSCALAR(pStrm, iCurrFNum, INT); @@ -1584,6 +1765,11 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) l = pThis->iCurrOffs; objSerializeSCALAR_VAR(pStrm, iCurrOffs, INT64, l); + l = pThis->inode; + objSerializeSCALAR_VAR(pStrm, inode, INT64, l); + + objSerializePTR(pStrm, prevLineSegment, PSZ); + CHKiRet(obj.EndSerialize(pStrm)); finalize_it: @@ -1681,6 +1867,8 @@ static rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) CHKiRet(strmSettOpenMode(pThis, pProp->val.num)); } else if(isProp("iCurrOffs")) { pThis->iCurrOffs = pProp->val.num; + } else if(isProp("inode")) { + pThis->inode = (ino_t) pProp->val.num; } else if(isProp("iMaxFileSize")) { CHKiRet(strmSetiMaxFileSize(pThis, pProp->val.num)); } else if(isProp("iMaxFiles")) { @@ -1689,6 +1877,8 @@ static rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) CHKiRet(strmSetiFileNumDigits(pThis, pProp->val.num)); } else if(isProp("bDeleteOnClose")) { CHKiRet(strmSetbDeleteOnClose(pThis, pProp->val.num)); + } else if(isProp("prevLineSegment")) { + CHKiRet(rsCStrConstructFromCStr(&pThis->prevLineSegment, pProp->val.pStr)); } finalize_it: @@ -1748,6 +1938,7 @@ CODESTARTobjQueryInterface(strm) pIf->GetCurrOffset = strmGetCurrOffset; pIf->Dup = strmDup; pIf->SetWCntr = strmSetWCntr; + pIf->CheckFileChange = CheckFileChange; /* set methods */ pIf->SetbDeleteOnClose = strmSetbDeleteOnClose; pIf->SetiMaxFileSize = strmSetiMaxFileSize; @@ -1757,11 +1948,14 @@ CODESTARTobjQueryInterface(strm) pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetbVeryReliableZip = strmSetbVeryReliableZip; pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; pIf->SetiSizeLimit = strmSetiSizeLimit; pIf->SetiFlushInterval = strmSetiFlushInterval; pIf->SetpszSizeLimitCmd = strmSetpszSizeLimitCmd; + pIf->Setcryprov = strmSetcryprov; + pIf->SetcryprovData = strmSetcryprovData; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index 0828f6fd..4f4a4301 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -41,7 +41,7 @@ * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); * -------------------------------------------------------------------------- * - * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -70,6 +70,7 @@ #include "glbl.h" #include "stream.h" #include "zlibw.h" +#include "cryprov.h" /* stream types */ typedef enum { @@ -112,6 +113,7 @@ typedef struct strm_s { int lenDir; int fd; /* the file descriptor, -1 if closed */ int fdDir; /* the directory's descriptor, in case bSync is requested (-1 if closed) */ + ino_t inode; /* current inode for files being monitored (undefined else) */ uchar *pszCurrFName; /* name of current file (if open) */ uchar *pIOBuf; /* the iobuffer currently in use to gather data */ size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ @@ -124,6 +126,8 @@ typedef struct strm_s { sbool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ sbool bStopWriter; /* shall writer thread terminate? */ sbool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ + sbool bzInitDone; /* did we do an init of zstrm already? */ + sbool bVeryReliableZip; /* shall we write interim headers to create a very reliable ZIP file? */ int iFlushInterval; /* flush in which interval - 0, no flushing */ pthread_mutex_t mut;/* mutex for flush in async mode */ pthread_cond_t notFull; @@ -131,7 +135,11 @@ typedef struct strm_s { pthread_cond_t isEmpty; unsigned short iEnq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ unsigned short iDeq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ + cryprov_if_t *cryprov; /* ptr to crypto provider; NULL = do not encrypt */ + void *cryprovData; /* opaque data ptr for provider use */ + void *cryprovFileData;/* opaque data ptr for file instance */ short iCnt; /* current nbr of elements in buffer */ + z_stream zstrm; /* zip stream to use */ struct { uchar *pBuf; size_t lenBuf; @@ -141,6 +149,7 @@ typedef struct strm_s { off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ sbool bIsTTY; /* is this a tty file? */ + cstr_t *prevLineSegment; /* for ReadLine, previous, unwritten part of file */ } strm_t; @@ -180,11 +189,23 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); /* v6 added */ rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr, int mode); + /* v7 added 2012-09-14 */ + INTERFACEpropSetMeth(strm, bVeryReliableZip, int); + /* v8 added 2013-03-21 */ + rsRetVal (*CheckFileChange)(strm_t *pThis); + /* v9 added 2013-04-04 */ + INTERFACEpropSetMeth(strm, cryprov, cryprov_if_t*); + INTERFACEpropSetMeth(strm, cryprovData, void*); ENDinterface(strm) -#define strmCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ +static inline int +strmGetCurrFileNum(strm_t *pStrm) { + return pStrm->iCurrFNum; +} /* prototypes */ PROTOTYPEObjClassInit(strm); +rsRetVal strmMultiFileSeek(strm_t *pThis, int fileNum, off64_t offs, off64_t *bytesDel); #endif /* #ifndef STREAM_H_INCLUDED */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index e7fd72c2..cb4f0457 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -32,6 +32,7 @@ #include <assert.h> #include <string.h> #include <ctype.h> +#include <stdarg.h> #include <sys/types.h> #include <libestr.h> #include "rsyslog.h" @@ -104,6 +105,55 @@ finalize_it: } +/* a helper function for rsCStr*Strf() + */ +static rsRetVal rsCStrConstructFromszStrv(cstr_t **ppThis, uchar *fmt, va_list ap) +{ + DEFiRet; + cstr_t *pThis; + va_list ap2; + int len; + + assert(ppThis != NULL); + + va_copy(ap2, ap); + len = vsnprintf(NULL, 0, (char*)fmt, ap2); + va_end(ap2); + + if(len < 0) + ABORT_FINALIZE(RS_RET_ERR); + + CHKiRet(rsCStrConstruct(&pThis)); + + pThis->iBufSize = pThis->iStrLen = len; + len++; /* account for the \0 written by vsnprintf */ + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * len)) == NULL) { + RSFREEOBJ(pThis); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + + vsnprintf((char*)pThis->pBuf, len, (char*)fmt, ap); + *ppThis = pThis; +finalize_it: + RETiRet; +} + + +/* construct from a printf-style formated string + */ +rsRetVal rsCStrConstructFromszStrf(cstr_t **ppThis, char *fmt, ...) +{ + DEFiRet; + va_list ap; + + va_start(ap, fmt); + iRet = rsCStrConstructFromszStrv(ppThis, (uchar*)fmt, ap); + va_end(ap); + + RETiRet; +} + + /* construct from es_str_t string * rgerhards 2010-12-03 */ @@ -256,6 +306,27 @@ rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) } +/* append a printf-style formated string + */ +rsRetVal rsCStrAppendStrf(cstr_t *pThis, uchar *fmt, ...) +{ + DEFiRet; + va_list ap; + cstr_t *pStr = NULL; + + va_start(ap, fmt); + iRet = rsCStrConstructFromszStrv(&pStr, fmt, ap); + va_end(ap); + + CHKiRet(iRet); + + iRet = cstrAppendCStr(pThis, pStr); + rsCStrDestruct(&pStr); +finalize_it: + RETiRet; +} + + rsRetVal rsCStrAppendInt(cstr_t *pThis, long i) { DEFiRet; @@ -482,6 +553,8 @@ rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis) register uchar *pC; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + if(pThis->iStrLen == 0) + goto done; /* empty string -> nothing to trim ;) */ i = pThis->iStrLen; pC = pThis->pBuf + i - 1; while(i > 0 && isspace((int)*pC)) { @@ -492,7 +565,7 @@ rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis) pThis->iStrLen = i; pThis->pBuf[pThis->iStrLen] = '0'; /* we always have this space */ - return RS_RET_OK; +done: return RS_RET_OK; } /* compare two string objects - works like strcmp(), but operates @@ -868,13 +941,7 @@ int rsCStrSzStrCmp(cstr_t *pCS1, uchar *psz, size_t iLenSz) * length, so we need to actually check if they * are equal. */ - register size_t i; - for(i = 0 ; i < iLenSz ; ++i) { - if(pCS1->pBuf[i] != psz[i]) - return pCS1->pBuf[i] - psz[i]; - } - /* if we arrive here, the strings are equal */ - return 0; + return strncmp((char*)pCS1->pBuf, (char*)psz, iLenSz); } else return pCS1->iStrLen - iLenSz; diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index bba004a0..d0502a5b 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -58,6 +58,7 @@ rsRetVal cstrConstruct(cstr_t **ppThis); rsRetVal cstrConstructFromESStr(cstr_t **ppThis, es_str_t *str); rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz); rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); +rsRetVal rsCStrConstructFromszStrf(cstr_t **ppThis, char *fmt, ...) __attribute__((format(printf,2, 3))); /** * Destruct the string buffer object. @@ -173,6 +174,12 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz); */ rsRetVal rsCStrAppendStrWithLen(cstr_t *pThis, uchar* psz, size_t iStrLen); +/** + * Append a printf-style formated string to the buffer. + * + * \param fmt pointer to the format string (see man 3 printf for details). Must not be NULL. + */ +rsRetVal rsCStrAppendStrf(cstr_t *pThis, uchar *fmt, ...); /** * Append an integer to the string. No special formatting is diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c index d14f0b37..2537e8d8 100644 --- a/runtime/strms_sess.c +++ b/runtime/strms_sess.c @@ -38,12 +38,14 @@ #include "errmsg.h" #include "netstrm.h" #include "msg.h" +#include "prop.h" #include "datetime.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) DEFobjCurrIf(errmsg) DEFobjCurrIf(netstrm) DEFobjCurrIf(datetime) @@ -86,7 +88,8 @@ CODESTARTobjDestruct(strms_sess) } /* now destruct our own properties */ free(pThis->fromHost); - free(pThis->fromHostIP); + if(pThis->fromHostIP != NULL) + prop.Destruct(&pThis->fromHostIP); ENDobjDestruct(strms_sess) @@ -111,17 +114,18 @@ SetHost(strms_sess_t *pThis, uchar *pszHost) RETiRet; } -/* set the remote host's IP. Note that the caller *hands over* the string. That is, +/* set the remote host's IP. Note that the caller *hands over* the property. That is, * the caller no longer controls it once SetHostIP() has received it. Most importantly, - * the caller must not free it. -- rgerhards, 2008-05-16 + * the caller must not destruct it. -- rgerhards, 2008-05-16 */ static rsRetVal -SetHostIP(strms_sess_t *pThis, uchar *pszHostIP) +SetHostIP(strms_sess_t *pThis, prop_t *ip) { DEFiRet; ISOBJ_TYPE_assert(pThis, strms_sess); - free(pThis->fromHostIP); - pThis->fromHostIP = pszHostIP; + if(pThis->fromHostIP != NULL) + prop.Destruct(&pThis->fromHostIP); + pThis->fromHostIP = ip; RETiRet; } @@ -188,8 +192,8 @@ Close(strms_sess_t *pThis) netstrm.Destruct(&pThis->pStrm); free(pThis->fromHost); pThis->fromHost = NULL; /* not really needed, but... */ - free(pThis->fromHostIP); - pThis->fromHostIP = NULL; /* not really needed, but... */ + if(pThis->fromHostIP != NULL) + prop.Destruct(&pThis->fromHostIP); RETiRet; } @@ -284,6 +288,7 @@ BEGINObjClassInit(strms_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ diff --git a/runtime/strms_sess.h b/runtime/strms_sess.h index 5c0309f8..86f692a8 100644 --- a/runtime/strms_sess.h +++ b/runtime/strms_sess.h @@ -33,9 +33,8 @@ struct strms_sess_s { strmsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ strmLstnPortList_t *pLstnInfo; /* pointer back to listener info */ netstrm_t *pStrm; -// uchar *pMsg; /* message (fragment) received */ uchar *fromHost; - uchar *fromHostIP; + prop_t *fromHostIP; void *pUsr; /* a user-pointer */ }; @@ -54,15 +53,17 @@ BEGINinterface(strms_sess) /* name must also be changed in ENDinterface macro! * rsRetVal (*SetUsrP)(strms_sess_t*, void*); void* (*GetUsrP)(strms_sess_t*); rsRetVal (*SetHost)(strms_sess_t *pThis, uchar*); - rsRetVal (*SetHostIP)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetHostIP)(strms_sess_t *pThis, prop_t*); rsRetVal (*SetStrm)(strms_sess_t *pThis, netstrm_t*); rsRetVal (*SetOnMsgReceive)(strms_sess_t *pThis, rsRetVal (*OnMsgReceive)(strms_sess_t*, uchar*, int)); ENDinterface(strms_sess) -#define strms_sessCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define strms_sessCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ /* interface changes * to version v2, rgerhards, 2009-05-22 * - Data structures changed * - SetLstnInfo entry point added + * version 3, rgerhads, 2013-01-21: + * - signature of SetHostIP() changed */ diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index 8310e832..e8b544b8 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -70,6 +70,7 @@ #include "netstrm.h" #include "nssel.h" #include "errmsg.h" +#include "prop.h" #include "unicode-helper.h" MODULE_TYPE_LIB @@ -89,6 +90,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) +DEFobjCurrIf(prop) /* forward definitions */ static rsRetVal create_strm_socket(strmsrv_t *pThis); @@ -418,7 +420,7 @@ SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSes int iSess = -1; struct sockaddr_storage *addr; uchar *fromHostFQDN = NULL; - uchar *fromHostIP = NULL; + prop_t *ip = NULL; ISOBJ_TYPE_assert(pThis, strmsrv); assert(pLstnInfo != NULL); @@ -444,7 +446,7 @@ SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSes /* get the host name */ CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); - CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); + CHKiRet(netstrm.GetRemoteIP(pNewStrm, &ip)); CHKiRet(netstrm.GetRemAddr(pNewStrm, &addr)); /* TODO: check if we need to strip the domain name here -- rgerhards, 2008-04-24 */ @@ -467,8 +469,8 @@ SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSes */ CHKiRet(strms_sess.SetHost(pSess, fromHostFQDN)); fromHostFQDN = NULL; /* we handed this string over */ - CHKiRet(strms_sess.SetHostIP(pSess, fromHostIP)); - fromHostIP = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetHostIP(pSess, ip)); + ip = NULL; /* we handed this string over */ CHKiRet(strms_sess.SetStrm(pSess, pNewStrm)); pNewStrm = NULL; /* prevent it from being freed in error handler, now done in strms_sess! */ CHKiRet(strms_sess.ConstructFinalize(pSess)); @@ -489,7 +491,8 @@ finalize_it: if(pNewStrm != NULL) netstrm.Destruct(&pNewStrm); free(fromHostFQDN); - free(fromHostIP); + if(ip != NULL) + prop.Destruct(&ip); } RETiRet; @@ -908,6 +911,7 @@ CODESTARTObjClassExit(strmsrv) objRelease(strms_sess, DONT_LOAD_LIB); objRelease(conf, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(netstrms, DONT_LOAD_LIB); objRelease(nssel, DONT_LOAD_LIB); @@ -930,6 +934,7 @@ BEGINObjClassInit(strmsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(strms_sess, DONT_LOAD_LIB)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, strmsrvDebugPrint); diff --git a/runtime/strmsrv.h b/runtime/strmsrv.h index 9ef28e47..f3d56d16 100644 --- a/runtime/strmsrv.h +++ b/runtime/strmsrv.h @@ -43,7 +43,7 @@ struct strmsrv_s { uchar *pszInputName; /**< value to be used as input name */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ int iLstnMax; /**< max nbr of listeners currently supported */ - netstrm_t **ppLstn; /**< our netstream listners */ + netstrm_t **ppLstn; /**< our netstream listeners */ strmLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ int iSessMax; /**< max number of sessions supported */ strmLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ diff --git a/runtime/typedefs.h b/runtime/typedefs.h index 4e7f1622..d3f68b4a 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -3,7 +3,7 @@ * * Begun 2010-11-25 RGerhards * - * Copyright (C) 2005-2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2005-2013 by Rainer Gerhards and Adiscon GmbH * * This file is part of the rsyslog runtime library. * @@ -25,6 +25,13 @@ */ #ifndef INCLUDED_TYPEDEFS_H #define INCLUDED_TYPEDEFS_H +#if defined(__FreeBSD__) +#include <sys/types.h> +#endif + +#ifndef HAVE_LSEEK64 +#include <unistd.h> +#endif /* some universal fixed size integer defines ... */ typedef long long int64; @@ -92,6 +99,10 @@ typedef struct cfgmodules_etry_s cfgmodules_etry_t; typedef struct outchannels_s outchannels_t; typedef struct modConfData_s modConfData_t; typedef struct instanceConf_s instanceConf_t; +typedef struct ratelimit_s ratelimit_t; +typedef struct action_s action_t; +typedef int rs_size_t; /* we do never need more than 2Gig strings, signed permits to + * use -1 as a special flag. */ typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ @@ -144,6 +155,10 @@ typedef enum { FIOP_ISEMPTY = 6 /* string empty <=> strlen(s) == 0 ?*/ } fiop_t; +#ifndef HAVE_LSEEK64 + typedef off_t off64_t; +#endif + /* types of configuration handlers */ typedef enum cslCmdHdlrType { @@ -162,6 +177,7 @@ typedef enum cslCmdHdlrType { eCmdHdlrSeverity, eCmdHdlrGetWord, eCmdHdlrString, + eCmdHdlrArray, eCmdHdlrQueueType, eCmdHdlrGoneAway /* statment existed, but is no longer supported */ } ecslCmdHdrlType; diff --git a/runtime/wtp.c b/runtime/wtp.c index a53a9888..19151e7c 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -381,6 +381,7 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in if(prctl(PR_SET_NAME, thrdName, 0, 0, 0) != 0) { DBGPRINTF("prctl failed, not setting thread name for '%s'\n", wtpGetDbgHdr(pThis)); } + dbgOutputTID((char*)thrdName); # endif pthread_cleanup_push(wtpWrkrExecCancelCleanup, pWti); |