From f62031df11317a9c861db464d9f95bea2f2d4596 Mon Sep 17 00:00:00 2001 From: Kaiwang Chen Date: Thu, 15 Mar 2012 10:58:40 +0100 Subject: added tool to recover disk queue if .qi file is missing (recover_qi.pl) --- ChangeLog | 2 + tools/Makefile.am | 3 +- tools/recover_qi.pl | 207 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 tools/recover_qi.pl diff --git a/ChangeLog b/ChangeLog index ef9d3735..c23f973e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 5.8.9 [V5-stable] 2012-03-?? +- added tool to recover disk queue if .qi file is missing (recover_qi.pl) + Thanks to Kaiwang Chen for contributing this tool - bugfix: stopped DA queue was never processed after a restart due to a regression from statistics module - added better doc for statsobj interface diff --git a/tools/Makefile.am b/tools/Makefile.am index 96657ad4..962ae507 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -47,4 +47,5 @@ zpipe_LDADD = -lz msggen_SOURCES = msggen.c endif -EXTRA_DIST = $(man_MANS) +EXTRA_DIST = $(man_MANS) \ + recover_qi.pl diff --git a/tools/recover_qi.pl b/tools/recover_qi.pl new file mode 100644 index 00000000..4e2cf9d5 --- /dev/null +++ b/tools/recover_qi.pl @@ -0,0 +1,207 @@ +#!/usr/bin/perl -w +# recover rsyslog disk queue index (.qi) from queue files (.nnnnnnnn). +# +# See: +# runtime/queue.c: qqueuePersist() +# runtime/queue.c: qqueueTryLoadPersistedInfo() +# +# kaiwang.chen@gmail.com 2012-03-14 +# +use strict; +use Getopt::Long; + +my %opt = (); +GetOptions(\%opt,"spool|w=s","basename|f=s","digits|d=i","help!"); +if ($opt{help}) { + print "Usage: +\t$0 -w WorkDirectory -f QueueFileName -d 8 > QueueFileName.qi +"; + exit; +} + +# runtime/queue.c: qConstructDisk() +my $iMaxFiles = 10000000; # 0+"1".( "0"x($opt{digits} - 1)); + +# get the list of queue files, spool directory excluded +my $re = qr/^\Q$opt{basename}\E\.\d{$opt{digits}}$/; +opendir(DIR, $opt{spool}) or die "can’t open spool: $!"; +my @qf = grep { /$re/ && -f "$opt{spool}/$_" } readdir(DIR); +closedir DIR; + +# ensure order and continuity +@qf = sort @qf; +my ($head) = ($qf[0] =~ /(\d+)$/); +my ($tail) = ($qf[-1] =~ /(\d+)$/); +$head += 0; +$tail += 0; +if ($tail-$head+1 != @qf || $tail > $iMaxFiles) { + die "broken queue: missing file(s) or wrong tail\n"; +} + +# collect some counters about the queue, assuming all are unprocessed entries. +my $sizeOnDisk = 0; +my $iQueueSize = 0; +chdir($opt{spool}) or die "can't chdir to spool: $!"; +print STDERR "traversing ". @qf ." files, please wait...\n"; +for (@qf) { + open FH, "<", $_ or die "can't read queue file $_\n"; + $sizeOnDisk += (stat FH)[7]; + while () { + $iQueueSize++ if /^new("qqueue",1); +$qqueue->property("iQueueSize", "INT", $iQueueSize); +$qqueue->property("tVars.disk.sizeOnDisk", "INT64", $sizeOnDisk); +$qqueue->property("tVars.disk.bytesRead", "INT64", 0); + +# runtime/stream.h: strmType_t +my $STREAMTYPE_FILE_CIRCULAR = 1; +# runtime/stream.h: strmMode_t +my $STREAMMODE_READ = 1; +my $STREAMMODE_WRITE_APPEND = 4; + +# runtime/stream.c: strmSerialize() +# write to end +my $strm_Write = Rsyslog::Obj->new("strm",1); +$strm_Write->property( "iCurrFNum", "INT", $tail); +$strm_Write->property( "pszFName", "PSZ", $opt{basename}); +$strm_Write->property( "iMaxFiles", "INT", $iMaxFiles); +$strm_Write->property( "bDeleteOnClose", "INT", 0); +$strm_Write->property( "sType", "INT", $STREAMTYPE_FILE_CIRCULAR); +$strm_Write->property("tOperationsMode", "INT", $STREAMMODE_WRITE_APPEND); +$strm_Write->property( "tOpenMode", "INT", 0600); +$strm_Write->property( "iCurrOffs","INT64", $iCurrOffs_Write); +# read from head +my $strm_ReadDel = Rsyslog::Obj->new("strm",1); +$strm_ReadDel->property( "iCurrFNum", "INT", $head); +$strm_ReadDel->property( "pszFName", "PSZ", $opt{basename}); +$strm_ReadDel->property( "iMaxFiles", "INT", $iMaxFiles); +$strm_ReadDel->property( "bDeleteOnClose", "INT", 1); +$strm_ReadDel->property( "sType", "INT", $STREAMTYPE_FILE_CIRCULAR); +$strm_ReadDel->property("tOperationsMode", "INT", $STREAMMODE_READ); +$strm_ReadDel->property( "tOpenMode", "INT", 0600); +$strm_ReadDel->property( "iCurrOffs","INT64", 0); + +# .qi +print $qqueue->serialize(); +print $strm_Write->serialize(); +print $strm_ReadDel->serialize(); + +exit; +#----------------------------------------------------------------------------- + +package Rsyslog::Serializable; +# runtime/obj.c +sub COOKIE_OBJLINE { '<' } +sub COOKIE_PROPLINE { '+' } +sub COOKIE_ENDLINE { '>' } +sub COOKIE_BLANKLINE { '.' } +# VARTYPE(short_ptype) +sub VARTYPE { + my ($t) = @_; + # runtime/obj-types.h: propType_t + my $ptype = "PROPTYPE_".$t; + # runtime/var.h: varType_t + my %vm = ( + VARTYPE_NONE => 0, + VARTYPE_STR => 1, + VARTYPE_NUMBER => 2, + VARTYPE_SYSLOGTIME => 3, + ); + # runtime/obj.c: SerializeProp() + my %p2v = ( + #PROPTYPE_NONE => "", + PROPTYPE_PSZ => "VARTYPE_STR", + PROPTYPE_SHORT => "VARTYPE_NUMBER", + PROPTYPE_INT => "VARTYPE_NUMBER", + PROPTYPE_LONG => "VARTYPE_NUMBER", + PROPTYPE_INT64 => "VARTYPE_NUMBER", + PROPTYPE_CSTR => "VARTYPE_STR", + #PROPTYPE_SYSLOGTIME => "VARTYPE_SYSLOGTIME", + ); + my $vtype = $p2v{$ptype}; + unless ($vtype) { + die "property type $t is not supported!\n"; + } + return $vm{$vtype}; +} +sub serialize { + my $self = shift; + # runtime/obj.c: objSerializeHeader() + my $x = COOKIE_OBJLINE(); + $x .= join(":", $self->type(), $self->cver(), $self->id(), $self->version()); + $x .= ":\n"; + for ( values %{$self->{props}} ) { + # runtime/obj.c: SerializeProp() + $x .= COOKIE_PROPLINE(); + $x .= join(":", + $_->{name}, + VARTYPE($_->{type}), + length($_->{value}), + $_->{value}); + $x .= ":\n"; + } + # runtime/obj.c: EndSerialize() + $x .= COOKIE_ENDLINE() . "End\n"; + $x .= COOKIE_BLANKLINE() . "\n"; +} +# constructor: new(id,version) +sub new { + my ($class, $id, $version) = @_; + $class = ref $class if ref $class; + bless { + id => $id, + version => $version, + props => {}, + }, $class; +} +sub id { + my $self = shift; + if (@_) { + my $x = $self->{id}; + $self->{id} = shift; + return $x; + } + return $self->{id}; +} +sub version { + my $self = shift; + if (@_) { + my $x = $self->{version}; + $self->{version} = shift; + return $x; + } + return $self->{version}; +} +# property(name, type, value) +sub property { + my $self = shift; + my $name = shift; + if (@_) { + my $x = $self->{props}{$name}; + $self->{props}{$name}{name} = $name; + $self->{props}{$name}{type} = shift; + $self->{props}{$name}{value} = shift; + return $x; + } + return $self->{props}{$name}; +} +1; +package Rsyslog::OPB; +use base qw(Rsyslog::Serializable); +sub type { 'OPB' } +sub cver { 1 } +sub new { shift->SUPER::new(@_) } +1; +package Rsyslog::Obj; +use base qw(Rsyslog::Serializable); +sub type { 'Obj' } +sub cver { 1 } +sub new { shift->SUPER::new(@_) } +1; -- cgit v1.2.3 From 43607c303cdaf5adc4983c559fab66942e98cbe5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 15 Mar 2012 11:30:33 +0100 Subject: preparing for 5.8.9 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c23f973e..98a1e285 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.8.9 [V5-stable] 2012-03-?? +Version 5.8.9 [V5-stable] 2012-03-15 - added tool to recover disk queue if .qi file is missing (recover_qi.pl) Thanks to Kaiwang Chen for contributing this tool - bugfix: stopped DA queue was never processed after a restart due to a diff --git a/configure.ac b/configure.ac index 3cb623d4..1f72bcce 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.8.8],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.8.9],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 54a04f80..690143de 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

-

This documentation is for version 5.8.8 (v5-stable branch) of rsyslog. +

This documentation is for version 5.8.9 (v5-stable branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

If you like rsyslog, you might -- cgit v1.2.3 From 59c99b2d8b23f8903e2130b7a149981afe1a8bd3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 17 Mar 2012 18:31:14 +0100 Subject: bugfix: memory leak in array passing output module mode --- ChangeLog | 3 +++ action.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 98a1e285..83e7f4ee 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 5.8.10 [V5-stable] 2012-??-?? +- bugfix: memory leak in array passing output module mode +--------------------------------------------------------------------------- Version 5.8.9 [V5-stable] 2012-03-15 - added tool to recover disk queue if .qi file is missing (recover_qi.pl) Thanks to Kaiwang Chen for contributing this tool diff --git a/action.c b/action.c index 278625ce..e859cce4 100644 --- a/action.c +++ b/action.c @@ -785,8 +785,9 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch) if(((uchar**)ppMsgs)[j] != NULL) { jArr = 0; while(ppMsgs[j][jArr] != NULL) { - d_free(ppMsgs[j][jArr++]); - ppMsgs[j][jArr++] = NULL; + d_free(ppMsgs[j][jArr]); + ppMsgs[j][jArr] = NULL; + ++jArr; } d_free(((uchar**)ppMsgs)[j]); ((uchar**)ppMsgs)[j] = NULL; -- cgit v1.2.3