From 7667845bd72b6f92eabc975318a4f288a77f2630 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Apr 2009 15:06:45 +0200 Subject: first attempt at dequeueing multiple batches inside the queue ... but this code has serious problems when terminating the queue, also it is far from being optimal. I will commit a series of patches (hopefully) as I am on the path to the final implementation. --- runtime/rsyslog.h | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 026fbbed..74ea5270 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -60,6 +60,7 @@ /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ +typedef struct aUsrp_s aUsrp_t; typedef struct thrdInfo thrdInfo_t; typedef struct obj_s obj_t; typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ -- cgit v1.2.3 From 10bab38993ae6853d7e23c6f6bd44eb0ed69e001 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Apr 2009 15:40:54 +0200 Subject: begin implementation of new transactional output module interface code is not complete, error cases are not handled. --- runtime/rsyslog.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 74ea5270..3b2dff62 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -58,6 +58,15 @@ #endif +/* the rsyslog core provides information about present feature to plugins + * asking it. Below are feature-test macros which must be used to query + * features. Note that this must be powers of two, so that multiple queries + * can be combined. -- rgerhards, 2009-04-27 + */ +#define CORE_FEATURE_BATCHING 1 +/*#define CORE_FEATURE_whatever 2 ... and so on ... */ + + /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ typedef struct aUsrp_s aUsrp_t; @@ -268,6 +277,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ + RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ + RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From bb79e96dc300fa5a2182e7c047afb3b15c5dc870 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 May 2009 15:27:40 +0200 Subject: moving to a cleaner implementation of batches ... now that we know what we need from a theoretical POV. --- runtime/rsyslog.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ee941b2b..53a510b3 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -95,6 +95,8 @@ typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; typedef struct vmstk_s vmstk_t; +typedef struct batch_obj_s batch_obj_t; +typedef struct batch_s batch_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ /* some universal 64 bit define... */ -- cgit v1.2.3 From e2b229868955a6f6a6380273314d0d90ddad1273 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 May 2009 17:57:04 +0200 Subject: action batch processing implemented ... passed initial tests, but of course more are needed --- runtime/rsyslog.h | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 53a510b3..25f9eefe 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -281,6 +281,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ + RS_RET_ACTION_FAILED = -2122, /**< action failed and is now suspended (consider this permanent for the time being) */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ /* RainerScript error messages (range 1000.. 1999) */ -- cgit v1.2.3 From 93f873277bfe5ebb309ff5e92f5dc7244ebd9f1a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 18 May 2009 17:28:34 +0200 Subject: t-delete list implemented, queue store drivers updated... ... on the way to the ultra-reliable queue modes (redesign doc). This version does not really work, but is a good commit point. Next comes queue size calculation. DA mode does not yet work. --- runtime/rsyslog.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 25f9eefe..3dd84ef6 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -67,6 +67,11 @@ /*#define CORE_FEATURE_whatever 2 ... and so on ... */ +/* some universal 64 bit define... */ +typedef long long int64; +typedef long long unsigned uint64; +typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ + /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ typedef struct aUsrp_s aUsrp_t; @@ -98,11 +103,8 @@ typedef struct vmstk_s vmstk_t; typedef struct batch_obj_s batch_obj_t; typedef struct batch_s batch_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ +typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ -/* some universal 64 bit define... */ -typedef long long int64; -typedef long long unsigned uint64; -typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ -- cgit v1.2.3 From 9f45b80ea9ea86d516c895d97fd8670df37e319e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 May 2009 15:12:49 +0200 Subject: free last processed message in all cases so far, the last processed message was only freed when the next one was processed. This has been changed now. More precisely, a better algorithm has been selected for the queue worker process, which also involves less overhead than the previous one. The fix for "free last processed message" as then more or less a side-effect (easy to do) of the new algorithm. --- runtime/rsyslog.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 3dd84ef6..01bbbb29 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -102,6 +102,7 @@ typedef struct tcpsrv_s tcpsrv_t; typedef struct vmstk_s vmstk_t; typedef struct batch_obj_s batch_obj_t; typedef struct batch_s batch_t; +typedef struct wtp_s wtp_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ @@ -290,10 +291,11 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ /* some generic error/status codes */ + RS_RET_OK = 0, /**< operation successful */ RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */ RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */ RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */ - RS_RET_OK = 0 /**< operation successful */ + RS_RET_IDLE = 4 /**< operation successful, but callee is idle (e.g. because queue is empty) */ }; /* some helpful macros to work with srRetVals. -- cgit v1.2.3 From aa9426f683fa6af9280bc63050ee0187ba4c57e1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 26 May 2009 12:43:43 +0200 Subject: solved design issue with queue termination ... and also improved the test suite. There is a design issue in the v3 queue engine that manifested to some serious problems with the new processing mode. However, in v3 shutdown may take eternally if a queue runs in DA mode, is configured to preserve data AND the action fails and retries immediately. There is no cure available for v3, it would require doing much of the work we have done on the new engine. The window of exposure, as one might guess from the description, is very small. That is probably the reason why we have not seen it in practice. --- runtime/rsyslog.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 0c671f03..a43c0327 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -298,7 +298,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */ RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */ RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */ - RS_RET_IDLE = 4 /**< operation successful, but callee is idle (e.g. because queue is empty) */ + RS_RET_IDLE = 4, /**< operation successful, but callee is idle (e.g. because queue is empty) */ + RS_RET_TERMINATE_WHEN_IDLE = 5 /**< operation successful, function is requested to terminate when idle */ }; /* some helpful macros to work with srRetVals. -- cgit v1.2.3 From d4564f8399f4362c7e79066370049f909cef996c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 May 2009 19:43:28 +0200 Subject: interim commit: working on failure cases slightly improved situation, would like to save it before carrying on --- runtime/rsyslog.h | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index a43c0327..58346d03 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -89,6 +89,7 @@ typedef struct nsd_gsspi_s nsd_gsspi_t; typedef struct nsd_nss_s nsd_nss_t; typedef struct nsdsel_ptcp_s nsdsel_ptcp_t; typedef struct nsdsel_gtls_s nsdsel_gtls_t; +typedef struct wti_s wti_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; typedef struct msg msg_t; -- cgit v1.2.3 From e6aeb1b995d6629ff04fd0f6068be2e9ffe9ca42 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 27 Jun 2009 12:31:38 +0200 Subject: fixed small compile issue --- runtime/rsyslog.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/rsyslog.h') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 3c8db169..76f320e6 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -81,7 +81,7 @@ typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ typedef char intTiny; /* 0..127! */ -typedef uchar uintTiny; /* 0..255! */ +typedef unsigned char uintTiny; /* 0..255! */ /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ -- cgit v1.2.3