From bb79e96dc300fa5a2182e7c047afb3b15c5dc870 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 May 2009 15:27:40 +0200 Subject: moving to a cleaner implementation of batches ... now that we know what we need from a theoretical POV. --- runtime/batch.h | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 runtime/batch.h (limited to 'runtime/batch.h') diff --git a/runtime/batch.h b/runtime/batch.h new file mode 100644 index 00000000..cb40cf42 --- /dev/null +++ b/runtime/batch.h @@ -0,0 +1,63 @@ +/* Definition of the batch_t data structure. + * I am not sure yet if this will become a full-blown object. For now, this header just + * includes the object definition and is not accompanied by code. + * + * Copyright 2009 by Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef BATCH_H_INCLUDED +#define BATCH_H_INCLUDED + +/* enum for batch states. Actually, we violate a layer here, in that we assume that a batch is used + * for action processing. So far, this seems acceptable, the status is simply ignored inside the + * main message queue. But over time, it could potentially be useful to split the two. + * rgerhad, 2009-05-12 + */ +typedef enum { + BATCH_STATE_RDY = 0, /* object ready for processing */ + BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ + BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unkonwn */ + BATCH_STATE_DISC = 3, /* discarded - processed OK, but do not submit to any other action */ +} batch_state_t; + + +/* an object inside a batch, including any information (state!) needed for it to "life". + */ +struct batch_obj_s { + obj_t *pUsrp; /* pointer to user object (most often message) */ + batch_state_t state; /* associated state */ +}; + +/* the batch + * This object is used to dequeue multiple user pointers which are than handed over + * to processing. The size of elements is fixed after queue creation, but may be + * modified by config variables (better said: queue properties). + * Note that a "user pointer" in rsyslog context so far always is a message + * object. We stick to the more generic term because queues may potentially hold + * other types of objects, too. + * rgerhards, 2009-05-12 + */ +struct batch_s { + int nElem; /* actual number of element in this entry */ + batch_obj_t *pElem; /* batch elements */ +}; + +#endif /* #ifndef BATCH_H_INCLUDED */ -- cgit v1.2.3 From e2b229868955a6f6a6380273314d0d90ddad1273 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 May 2009 17:57:04 +0200 Subject: action batch processing implemented ... passed initial tests, but of course more are needed --- runtime/batch.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'runtime/batch.h') diff --git a/runtime/batch.h b/runtime/batch.h index cb40cf42..fcbbafce 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -35,7 +35,8 @@ typedef enum { BATCH_STATE_RDY = 0, /* object ready for processing */ BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unkonwn */ - BATCH_STATE_DISC = 3, /* discarded - processed OK, but do not submit to any other action */ + BATCH_STATE_COMM = 3, /* message successfully commited */ + BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */ } batch_state_t; @@ -57,6 +58,7 @@ struct batch_obj_s { */ struct batch_s { int nElem; /* actual number of element in this entry */ + int iDoneUpTo; /* all messages below this index have state other than RDY */ batch_obj_t *pElem; /* batch elements */ }; -- cgit v1.2.3 From 93f873277bfe5ebb309ff5e92f5dc7244ebd9f1a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 18 May 2009 17:28:34 +0200 Subject: t-delete list implemented, queue store drivers updated... ... on the way to the ultra-reliable queue modes (redesign doc). This version does not really work, but is a good commit point. Next comes queue size calculation. DA mode does not yet work. --- runtime/batch.h | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime/batch.h') diff --git a/runtime/batch.h b/runtime/batch.h index fcbbafce..eb266b3f 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -59,6 +59,7 @@ struct batch_obj_s { struct batch_s { int nElem; /* actual number of element in this entry */ int iDoneUpTo; /* all messages below this index have state other than RDY */ + qDeqID deqID; /* ID of dequeue operation that generated this batch */ batch_obj_t *pElem; /* batch elements */ }; -- cgit v1.2.3 From 0cf8e88a348dc574244e4f5c2be26f47e8bfff08 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 19 May 2009 18:58:33 +0200 Subject: solved the intended-discard-during-dequeue issue --- runtime/batch.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'runtime/batch.h') diff --git a/runtime/batch.h b/runtime/batch.h index eb266b3f..031718a7 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -55,9 +55,15 @@ struct batch_obj_s { * object. We stick to the more generic term because queues may potentially hold * other types of objects, too. * rgerhards, 2009-05-12 + * Note that nElem is not necessarily equal to nElemDeq. This is the case when we + * discard some elements (because of configuration) during dequeue processing. As + * all Elements are only deleted when the batch is processed, we can not immediately + * delete them. So we need to keep their number that we can delete them when the batch + * is completed (else, the whole process does not work correctly). */ struct batch_s { int nElem; /* actual number of element in this entry */ + int nElemDeq; /* actual number of elements dequeued (and thus to be deleted) - see comment above! */ int iDoneUpTo; /* all messages below this index have state other than RDY */ qDeqID deqID; /* ID of dequeue operation that generated this batch */ batch_obj_t *pElem; /* batch elements */ -- cgit v1.2.3