Rev 1 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed
#! /usr/bin/perl
my @errors;
my @fileList;
my %backupDomains; # loaded from configuration file
my $MISSING_BACKUP_MAILTO; # loaded from configuration file
my $TESTING = 0;
# standard find and load configuration file
sub loadConfigurationFile {
my $configuration_file = shift;
use File::Basename;
use Cwd qw(realpath);
my $filename = realpath($0); # get my real path
my $directories;
my $suffix;
#print "$configuration_file\n";
$configuration_file = $filename unless $configuration_file;
#print "$configuration_file\n";
if ( $configuration_file !~ m/\// ) { # no path information
($filename, $directories, $suffix) = fileparse($filename,qr/\.[^.]*/); # break filename apart
#print "No Path Given\n";
} else {
($filename, $directories, $suffix) = fileparse($configuration_file,qr/\.[^.]*/); # break filename apart
$configuration_file = '';
#print "Path included\n";
}
unless (-e $directories . ($configuration_file ? $configuration_file : $filename) . '.conf' ) {
$lookingIn = $directories;
while ($lookingIn) {
$lookingIn =~ m/^(.*\/)[^\/]+\//;
$lookingIn = $1;
#print "$lookingIn\n";
if (-e $lookingIn . ($configuration_file ? $configuration_file : $filename) . '.conf' ) {
$directories = $lookingIn;
$lookingIn = '';
}
}
}
$configuration_file = $directories . ($configuration_file ? $configuration_file : $filename) . '.conf'; # add the .conf
# print "$configuration_file\n";
# die;
open CONFFILE, "<$configuration_file" or die "Can not open configuration file $configuration_file";
my $confFileContents = join( '', <CONFFILE> );
close CONFFILE;
return $confFileContents;
}
sub processFile {
my ($filename,$deviceID, $reportDate) = @_;
return '' unless $filename and $deviceID and $reportDate;
open DATA, "<$filename" or die "Could not open $filename: $!";
my $line;
my %stats;
my $backupVersion;
# get header
print "Starting $filename\n";
$stats{'version'} = 'unk';
while (($line = <DATA>)) {
last if ($line =! m/file list/);
if ($line =~ m/backup v([0-9a-z.]+)/) {
$stats{'version'} = qq/'$1'/;
}
}
my $linecount = 0;
while ($line = <DATA>) {
chomp $line;
last if $line !~ m/^\s*$/; # on non-blank line exit after
}
if ($line !~ m/^Backup Version/ ) { # we found actual file transfer stuff
#print "\tProcessing internal lines, previous was\n\t$line\n";
while ($line = <DATA>) {
$linecount++;
last if $line =~ m/^\s*$/;
last if $line =~ m/^(Number of Files:)|(Backup Version)/;
if ($line =~ m/^deleting/i ) {
$stats{'files_deleted'}++;
} elsif ($line !~ m/\/$/) { # not a directory, but a real file
$stats{'updated'}++;
}
}
}
$stats{'files_deleted'} = 0 unless defined $stats{'files_deleted'};
$stats{'files_count'} = $linecount;
foreach $key ('files_size','transferred_count','transferred_size','data_received', 'data_sent','StorageSize') {
$stats{$key} = 0 unless defined $stats{$key};
}
while ($line = <DATA>) {
chomp $line;
if ($line =~ m/Number of files:\s+(\d+)/) {
$stats{'files_count'} = $1;
} elsif ($line =~ m/Number of files transferred:\s+(\d+)/) {
$stats{'transferred_count'} = $1;
} elsif ($line =~ m/Total file size:\s+(\d+)\s+bytes/) {
$stats{'files_size'} = $1;
} elsif ($line =~ m/Total transferred file size:\s+(\d+)\s+bytes/) {
$stats{'transferred_size'} = $1;
} elsif ($line =~ m/sent\s+(\d+)\s+bytes\s+received\s+(\d+)\s+bytes\s+([0-9.]+)\s+bytes\/sec/) {
$stats{'data_sent'} = $1;
$stats{'data_received'} = $2;
} elsif ($line =~ m/^Begin\s+(\d+)\s*\[(\d+)\]/){
$stats{'start_time'} = qq/'$1'/;
} elsif ($line =~ m/^Complete\s+(\d+)\s*\[(\d+)\]/){
$stats{'end_time'} = qq/'$1'/;
} elsif ($line =~ m/^Duration\s+([\d:]+)\s+\[(\d+) seconds\]/){
$stats{'Duration'} = $2;
} elsif ($line =~ m/^(\d+)\[(\d+)\] began/ ) {
$stats{'start_time'} = qq/'$1'/;
} elsif ($line =~ m/^(\d+)\[(\d+)\] finished/ ) {
$stats{'end_time'} = qq/'$1'/;
} elsif ($line =~ m/Backup Version ([0-9.]+)/) {
$stats{'version'} = qq/'$1'/;
} elsif ($line =~ m/Directory Size On Server: ([0-9.]+) bytes/ ) {
$stats{'StorageSize'} = $1;
}
}
# make sure all columns have a value
if ($stats{'start_time'} && $stats{'end_time'} && $stats{'version'}) {
return qq/insert into backups_run (backups_id,report_date,start_time,end_time,version,
files_count,files_size,transferred_count,
transferred_size,files_deleted,
data_sent,data_received,disk_used)
select backups_id,
'$reportDate',
$stats{'start_time'},
$stats{'end_time'},
$stats{'version'},
$stats{'files_count'},
$stats{'files_size'},
$stats{'transferred_count'},
$stats{'transferred_size'},
$stats{'files_deleted'},
$stats{'data_sent'},
$stats{'data_received'},
$stats{'StorageSize'}
from backups
where device_id = $deviceID/;
} else {
print "Error Processing $filename\n";
foreach $key ('start_time','end_time','version','files_count','files_size','transferred_count','transferred_size','files_deleted','data_sent','data_received') {
print "\t$key = [$stats{$key}]\n";
}
print "\n";
return '';
}
}
sub getAttachments{
opendir (DATADIR,$sourceDir) || die "can't opendir $sourceDir: $!"; # read the directory
#print "Opened $sourceDir\n";
while ( my $thisFile = readdir(DATADIR) ) { # for each file
next if $thisFile =~ m/^\./; # skip if it is a "dot" file
#print "It is not a dot file\n";
next unless -f "$sourceDir/$thisFile";
$thisFile = $sourceDir . '/' . $thisFile; # ok, get fully qualified path/filename
#print "Preparing $thisFile\n";
`munpack -f -C $tempDir < '$thisFile'`;
&postProcess($thisFile);
}
closedir(DATADIR);
`rm -f $tempDir/*.desc`;
`gunzip -f $tempDir/*.gz`;
}
sub getDeviceID {
my $device = shift;
$device = &GenericSQL::fixStringValue($dbh, $device);
my $sql = "select distinct device_id from device where name = $device union select device_id from device_alias where alias = $device";
#print "\t$sql\n";
#return 1;
return &GenericSQL::getOneValue($dbh,$sql);
}
sub parseFileName {
my $fileName = shift;
$fileName =~ m/(\d+)_(.+)\.backup\.log/;
return (&getDeviceID($2),$2,$1);
}
###############################################################################
# BEGIN MAIN ROUTINE
###############################################################################
BEGIN{
# load the configuration file
eval ( &loadConfigurationFile );
push @INC, $LIBRARIES;
}
use strict;
no strict 'vars';
use Data::Dumper;
use GenericSQL; # generic, home grown MySQL access routines
#use GenericTemplates;
use Logging; # generic, home grown logging routines
use Date::Format; # allows us to format our dates nicely
use Date::Parse; # VERY KEWL, parses out a huge number of date formats
$dbh = DBI->connect( $DSN, $DB_USER , $DB_PASS ) or die $DBI::errstr; # try to connect to db first
&getAttachments; # unpacks all attachments into $tempDir and gunzips them
# now, process each file in turn
opendir (DATADIR,$tempDir) || die "can't opendir $tempDir: $!"; # read the directory
while ( my $thisFile = readdir(DATADIR) ) { # for each file
next if $thisFile =~ m/^\./; # skip if it is a "dot" file
print "It is not a dot file\n" if $TESTING >= 3;
next unless -f "$tempDir/$thisFile";
print "Looking at $thisFile\n" if $TESTING >= 2;
($device_id,$device,$report_date) = &parseFileName($thisFile);
print "\tparseFileName = Device: $device_id Date:$report_date\n" if $TESTING >= 2;
unless ( $device_id ) {
push @errors, "Could not find device info on $thisFile";
next;
}
unless ( $report_date ) {
push @errors, "Invalid report date for $thisFile";
next;
}
if ( exists( $backupDomains{ $device } ) ) {
$backupDomains{ $device } = 1;
} else {
push @errors, "$device not created in backups to process, processing anyway";
}
print "\tDevice: $device_id\tDate: $report_date\n" if $TESTING;
# check for duplicate report
next if &GenericSQL::getOneValue($dbh,qq/select count(*) from backups_run join backups using (backups_id) where device_id = $device_id and report_date = '$reportDate'/);
$sql = &processFile($tempDir . '/' . $thisFile, $device_id,$report_date);
# if no sql returned, it is not a valid file
unless ( $sql ) {
push @errors, "Could not generate SQL statement for $thisFile";
next;
}
unlink "$tempDir/$thisFile";
print "$sql\n" if $TESTING >= 3;
&GenericSQL::doSQL( $dbh,$sql );
}
closedir(DATADIR);
foreach my $temp ( keys %backupDomains ) {
push @errors, "No backup for $temp" unless $backupDomains{$temp};
}
if ( @errors ) {
print "ERRORS DETECTED:\n" . join( "\n", @errors ) . "\n";
my $message = "Subject: ERRORS on backups\n\n" . join( "\n", @errors ) . "\n";
open SENDMAIL, "|$SENDMAIL" or die "Can not open $SENDMAIL: $!\n";
print SENDMAIL "To: $MISSING_BACKUP_MAILTO\n";
print SENDMAIL $message;
close SENDMAIL;
}
1;