From f62031df11317a9c861db464d9f95bea2f2d4596 Mon Sep 17 00:00:00 2001 From: Kaiwang Chen Date: Thu, 15 Mar 2012 10:58:40 +0100 Subject: added tool to recover disk queue if .qi file is missing (recover_qi.pl) --- ChangeLog | 2 + tools/Makefile.am | 3 +- tools/recover_qi.pl | 207 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 tools/recover_qi.pl diff --git a/ChangeLog b/ChangeLog index ef9d3735..c23f973e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 5.8.9 [V5-stable] 2012-03-?? +- added tool to recover disk queue if .qi file is missing (recover_qi.pl) + Thanks to Kaiwang Chen for contributing this tool - bugfix: stopped DA queue was never processed after a restart due to a regression from statistics module - added better doc for statsobj interface diff --git a/tools/Makefile.am b/tools/Makefile.am index 96657ad4..962ae507 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -47,4 +47,5 @@ zpipe_LDADD = -lz msggen_SOURCES = msggen.c endif -EXTRA_DIST = $(man_MANS) +EXTRA_DIST = $(man_MANS) \ + recover_qi.pl diff --git a/tools/recover_qi.pl b/tools/recover_qi.pl new file mode 100644 index 00000000..4e2cf9d5 --- /dev/null +++ b/tools/recover_qi.pl @@ -0,0 +1,207 @@ +#!/usr/bin/perl -w +# recover rsyslog disk queue index (.qi) from queue files (.nnnnnnnn). +# +# See: +# runtime/queue.c: qqueuePersist() +# runtime/queue.c: qqueueTryLoadPersistedInfo() +# +# kaiwang.chen@gmail.com 2012-03-14 +# +use strict; +use Getopt::Long; + +my %opt = (); +GetOptions(\%opt,"spool|w=s","basename|f=s","digits|d=i","help!"); +if ($opt{help}) { + print "Usage: +\t$0 -w WorkDirectory -f QueueFileName -d 8 > QueueFileName.qi +"; + exit; +} + +# runtime/queue.c: qConstructDisk() +my $iMaxFiles = 10000000; # 0+"1".( "0"x($opt{digits} - 1)); + +# get the list of queue files, spool directory excluded +my $re = qr/^\Q$opt{basename}\E\.\d{$opt{digits}}$/; +opendir(DIR, $opt{spool}) or die "can’t open spool: $!"; +my @qf = grep { /$re/ && -f "$opt{spool}/$_" } readdir(DIR); +closedir DIR; + +# ensure order and continuity +@qf = sort @qf; +my ($head) = ($qf[0] =~ /(\d+)$/); +my ($tail) = ($qf[-1] =~ /(\d+)$/); +$head += 0; +$tail += 0; +if ($tail-$head+1 != @qf || $tail > $iMaxFiles) { + die "broken queue: missing file(s) or wrong tail\n"; +} + +# collect some counters about the queue, assuming all are unprocessed entries. +my $sizeOnDisk = 0; +my $iQueueSize = 0; +chdir($opt{spool}) or die "can't chdir to spool: $!"; +print STDERR "traversing ". @qf ." files, please wait...\n"; +for (@qf) { + open FH, "<", $_ or die "can't read queue file $_\n"; + $sizeOnDisk += (stat FH)[7]; + while () { + $iQueueSize++ if /^new("qqueue",1); +$qqueue->property("iQueueSize", "INT", $iQueueSize); +$qqueue->property("tVars.disk.sizeOnDisk", "INT64", $sizeOnDisk); +$qqueue->property("tVars.disk.bytesRead", "INT64", 0); + +# runtime/stream.h: strmType_t +my $STREAMTYPE_FILE_CIRCULAR = 1; +# runtime/stream.h: strmMode_t +my $STREAMMODE_READ = 1; +my $STREAMMODE_WRITE_APPEND = 4; + +# runtime/stream.c: strmSerialize() +# write to end +my $strm_Write = Rsyslog::Obj->new("strm",1); +$strm_Write->property( "iCurrFNum", "INT", $tail); +$strm_Write->property( "pszFName", "PSZ", $opt{basename}); +$strm_Write->property( "iMaxFiles", "INT", $iMaxFiles); +$strm_Write->property( "bDeleteOnClose", "INT", 0); +$strm_Write->property( "sType", "INT", $STREAMTYPE_FILE_CIRCULAR); +$strm_Write->property("tOperationsMode", "INT", $STREAMMODE_WRITE_APPEND); +$strm_Write->property( "tOpenMode", "INT", 0600); +$strm_Write->property( "iCurrOffs","INT64", $iCurrOffs_Write); +# read from head +my $strm_ReadDel = Rsyslog::Obj->new("strm",1); +$strm_ReadDel->property( "iCurrFNum", "INT", $head); +$strm_ReadDel->property( "pszFName", "PSZ", $opt{basename}); +$strm_ReadDel->property( "iMaxFiles", "INT", $iMaxFiles); +$strm_ReadDel->property( "bDeleteOnClose", "INT", 1); +$strm_ReadDel->property( "sType", "INT", $STREAMTYPE_FILE_CIRCULAR); +$strm_ReadDel->property("tOperationsMode", "INT", $STREAMMODE_READ); +$strm_ReadDel->property( "tOpenMode", "INT", 0600); +$strm_ReadDel->property( "iCurrOffs","INT64", 0); + +# .qi +print $qqueue->serialize(); +print $strm_Write->serialize(); +print $strm_ReadDel->serialize(); + +exit; +#----------------------------------------------------------------------------- + +package Rsyslog::Serializable; +# runtime/obj.c +sub COOKIE_OBJLINE { '<' } +sub COOKIE_PROPLINE { '+' } +sub COOKIE_ENDLINE { '>' } +sub COOKIE_BLANKLINE { '.' } +# VARTYPE(short_ptype) +sub VARTYPE { + my ($t) = @_; + # runtime/obj-types.h: propType_t + my $ptype = "PROPTYPE_".$t; + # runtime/var.h: varType_t + my %vm = ( + VARTYPE_NONE => 0, + VARTYPE_STR => 1, + VARTYPE_NUMBER => 2, + VARTYPE_SYSLOGTIME => 3, + ); + # runtime/obj.c: SerializeProp() + my %p2v = ( + #PROPTYPE_NONE => "", + PROPTYPE_PSZ => "VARTYPE_STR", + PROPTYPE_SHORT => "VARTYPE_NUMBER", + PROPTYPE_INT => "VARTYPE_NUMBER", + PROPTYPE_LONG => "VARTYPE_NUMBER", + PROPTYPE_INT64 => "VARTYPE_NUMBER", + PROPTYPE_CSTR => "VARTYPE_STR", + #PROPTYPE_SYSLOGTIME => "VARTYPE_SYSLOGTIME", + ); + my $vtype = $p2v{$ptype}; + unless ($vtype) { + die "property type $t is not supported!\n"; + } + return $vm{$vtype}; +} +sub serialize { + my $self = shift; + # runtime/obj.c: objSerializeHeader() + my $x = COOKIE_OBJLINE(); + $x .= join(":", $self->type(), $self->cver(), $self->id(), $self->version()); + $x .= ":\n"; + for ( values %{$self->{props}} ) { + # runtime/obj.c: SerializeProp() + $x .= COOKIE_PROPLINE(); + $x .= join(":", + $_->{name}, + VARTYPE($_->{type}), + length($_->{value}), + $_->{value}); + $x .= ":\n"; + } + # runtime/obj.c: EndSerialize() + $x .= COOKIE_ENDLINE() . "End\n"; + $x .= COOKIE_BLANKLINE() . "\n"; +} +# constructor: new(id,version) +sub new { + my ($class, $id, $version) = @_; + $class = ref $class if ref $class; + bless { + id => $id, + version => $version, + props => {}, + }, $class; +} +sub id { + my $self = shift; + if (@_) { + my $x = $self->{id}; + $self->{id} = shift; + return $x; + } + return $self->{id}; +} +sub version { + my $self = shift; + if (@_) { + my $x = $self->{version}; + $self->{version} = shift; + return $x; + } + return $self->{version}; +} +# property(name, type, value) +sub property { + my $self = shift; + my $name = shift; + if (@_) { + my $x = $self->{props}{$name}; + $self->{props}{$name}{name} = $name; + $self->{props}{$name}{type} = shift; + $self->{props}{$name}{value} = shift; + return $x; + } + return $self->{props}{$name}; +} +1; +package Rsyslog::OPB; +use base qw(Rsyslog::Serializable); +sub type { 'OPB' } +sub cver { 1 } +sub new { shift->SUPER::new(@_) } +1; +package Rsyslog::Obj; +use base qw(Rsyslog::Serializable); +sub type { 'Obj' } +sub cver { 1 } +sub new { shift->SUPER::new(@_) } +1; -- cgit v1.2.3 From 43607c303cdaf5adc4983c559fab66942e98cbe5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 15 Mar 2012 11:30:33 +0100 Subject: preparing for 5.8.9 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c23f973e..98a1e285 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.8.9 [V5-stable] 2012-03-?? +Version 5.8.9 [V5-stable] 2012-03-15 - added tool to recover disk queue if .qi file is missing (recover_qi.pl) Thanks to Kaiwang Chen for contributing this tool - bugfix: stopped DA queue was never processed after a restart due to a diff --git a/configure.ac b/configure.ac index 3cb623d4..1f72bcce 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.8.8],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.8.9],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 54a04f80..690143de 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

-

This documentation is for version 5.8.8 (v5-stable branch) of rsyslog. +

This documentation is for version 5.8.9 (v5-stable branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

If you like rsyslog, you might -- cgit v1.2.3 From 59c99b2d8b23f8903e2130b7a149981afe1a8bd3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 17 Mar 2012 18:31:14 +0100 Subject: bugfix: memory leak in array passing output module mode --- ChangeLog | 3 +++ action.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 98a1e285..83e7f4ee 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 5.8.10 [V5-stable] 2012-??-?? +- bugfix: memory leak in array passing output module mode +--------------------------------------------------------------------------- Version 5.8.9 [V5-stable] 2012-03-15 - added tool to recover disk queue if .qi file is missing (recover_qi.pl) Thanks to Kaiwang Chen for contributing this tool diff --git a/action.c b/action.c index 278625ce..e859cce4 100644 --- a/action.c +++ b/action.c @@ -785,8 +785,9 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch) if(((uchar**)ppMsgs)[j] != NULL) { jArr = 0; while(ppMsgs[j][jArr] != NULL) { - d_free(ppMsgs[j][jArr++]); - ppMsgs[j][jArr++] = NULL; + d_free(ppMsgs[j][jArr]); + ppMsgs[j][jArr] = NULL; + ++jArr; } d_free(((uchar**)ppMsgs)[j]); ((uchar**)ppMsgs)[j] = NULL; -- cgit v1.2.3 From 948d8231c7d4d47bb25ef6fc83d9a5ac6c8576a6 Mon Sep 17 00:00:00 2001 From: Tomas Heinrich Date: Thu, 5 Apr 2012 13:55:20 +0200 Subject: bugfix: segfault if disk-queue was started up with old queue file Signed-off-by: Rainer Gerhards --- ChangeLog | 2 ++ runtime/queue.c | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 83e7f4ee..b19dba0e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 5.8.10 [V5-stable] 2012-??-?? +- bugfix: segfault if disk-queue was started up with old queue file + Thanks to Tomas Heinrich for the patch. - bugfix: memory leak in array passing output module mode --------------------------------------------------------------------------- Version 5.8.9 [V5-stable] 2012-03-15 diff --git a/runtime/queue.c b/runtime/queue.c index 621a4eed..137c9547 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2018,13 +2018,16 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) CHKiRet(obj.EndSerialize(psQIF)); /* now persist the stream info */ - CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); - CHKiRet(strm.Serialize(pThis->tVars.disk.pReadDel, psQIF)); + if(pThis->tVars.disk.pWrite != NULL) + CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); + if(pThis->tVars.disk.pReadDel != NULL) + CHKiRet(strm.Serialize(pThis->tVars.disk.pReadDel, psQIF)); /* tell the input file object that it must not delete the file on close if the queue * is non-empty - but only if we are not during a simple checkpoint */ - if(bIsCheckpoint != QUEUE_CHECKPOINT) { + if(bIsCheckpoint != QUEUE_CHECKPOINT + && pThis->tVars.disk.pReadDel != NULL) { CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 0)); } -- cgit v1.2.3 From 2b76241630aa93218c8853a110251dd71478429e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 5 Apr 2012 13:57:26 +0200 Subject: doc: noted disk queue format change from v4->v5 --- doc/v5compatibility.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/v5compatibility.html b/doc/v5compatibility.html index 6d60062f..cc875b7b 100644 --- a/doc/v5compatibility.html +++ b/doc/v5compatibility.html @@ -16,6 +16,11 @@ available. This processing was redundant and had a lot a drawbacks. For details, please see the rsyslog v4 compatibility notes which elaborate on the reasons and the (few) things you may need to change. +

Queue on-disk format

+

The queue information file format has been changed. When upgrading from v4 to +v5, make sure that the queue is emptied and no on-disk structure present. We did +not go great length in understanding the old format, as there was too little demand +for that (and it being quite some effort if done right).

Queue Worker Thread Shutdown

Previous rsyslog versions had the capability to "run" on zero queue worker if no work was required. This was done to save a very limited number of resources. However, -- cgit v1.2.3 From a192300aa26484c88aedd306fdb92d7752eee427 Mon Sep 17 00:00:00 2001 From: Tomas Heinrich Date: Thu, 5 Apr 2012 14:18:11 +0200 Subject: bugfix: segfault on startup if $actionqueuefilename was missing for disk queue config Signed-off-by: Rainer Gerhards --- ChangeLog | 3 +++ runtime/queue.c | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b19dba0e..908dfda9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,8 @@ --------------------------------------------------------------------------- Version 5.8.10 [V5-stable] 2012-??-?? +- bugfix: segfault on startup if $actionqueuefilename was missing for disk + queue config + Thanks to Tomas Heinrich for the patch. - bugfix: segfault if disk-queue was started up with old queue file Thanks to Tomas Heinrich for the patch. - bugfix: memory leak in array passing output module mode diff --git a/runtime/queue.c b/runtime/queue.c index 137c9547..9f318523 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2119,7 +2119,8 @@ CODESTARTobjDestruct(qqueue) * direct queue - because in both cases we have none... ;) * with a child! -- rgerhards, 2008-01-28 */ - if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) + if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL + && pThis->pWtpReg != NULL) ShutdownWorkers(pThis); if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { -- cgit v1.2.3