Subversion Repositories computer_asset_manager_v1

#! /usr/bin/perl

my @errors;

my @fileList;

my %backupDomains; # loaded from configuration file

my $MISSING_BACKUP_MAILTO; # loaded from configuration file

my $TESTING = 0;

# standard find and load configuration file

sub loadConfigurationFile {

   my $configuration_file = shift;

   use File::Basename;

   use Cwd qw(realpath);

   my $filename  = realpath($0); # get my real path

   my $directories;

   my $suffix;

   #print "$configuration_file\n";

   $configuration_file = $filename unless $configuration_file;

   #print "$configuration_file\n";

   if ( $configuration_file !~ m/\// ) { # no path information

      ($filename, $directories, $suffix) = fileparse($filename,qr/\.[^.]*/); # break filename apart

      #print "No Path Given\n";

   } else {

      ($filename, $directories, $suffix) = fileparse($configuration_file,qr/\.[^.]*/); # break filename apart

      $configuration_file = '';

      #print "Path included\n";

   }

   unless (-e $directories . ($configuration_file ? $configuration_file : $filename) . '.conf' ) {

      $lookingIn = $directories;

      while ($lookingIn) {

         $lookingIn =~ m/^(.*\/)[^\/]+\//;

         $lookingIn = $1;

         #print "$lookingIn\n";

         if (-e $lookingIn . ($configuration_file ? $configuration_file : $filename) . '.conf' ) {

            $directories = $lookingIn;

            $lookingIn = '';

         }

      }

   }

   $configuration_file = $directories . ($configuration_file ? $configuration_file : $filename) . '.conf'; # add the .conf

#   print "$configuration_file\n";

#   die;

   open CONFFILE, "<$configuration_file" or die "Can not open configuration file $configuration_file";

   my $confFileContents = join( '', <CONFFILE> );

   close CONFFILE;

   return $confFileContents;

}

sub processFile {

   my ($filename,$deviceID, $reportDate) = @_;

   return '' unless $filename and $deviceID and $reportDate;

   open DATA, "<$filename" or die "Could not open $filename: $!";

   my $line;

   my %stats;

   my $backupVersion;

   # get header

   print "Starting $filename\n";

   $stats{'version'} = 'unk';

   while (($line = <DATA>)) {

      last if ($line =! m/file list/);

      if ($line =~ m/backup v([0-9a-z.]+)/) {

         $stats{'version'} = qq/'$1'/;

      }

   }

   my $linecount = 0;

   while ($line = <DATA>) {

      chomp $line;

      last if $line !~ m/^\s*$/; # on non-blank line exit after 

   }

   if ($line !~ m/^Backup Version/ ) { # we found actual file transfer stuff

      #print "\tProcessing internal lines, previous was\n\t$line\n";

      while ($line = <DATA>) {

         $linecount++;

         last if $line =~ m/^\s*$/;

         last if $line =~ m/^(Number of Files:)|(Backup Version)/;

         if ($line =~ m/^deleting/i ) {

         $stats{'files_deleted'}++;

         } elsif ($line !~ m/\/$/) { # not a directory, but a real file

         $stats{'updated'}++;

         }

      }

   }

   $stats{'files_deleted'} = 0 unless defined $stats{'files_deleted'};

   $stats{'files_count'} = $linecount;

   foreach $key ('files_size','transferred_count','transferred_size','data_received', 'data_sent','StorageSize') {

      $stats{$key} = 0 unless defined $stats{$key};

   }

   while ($line = <DATA>) {

      chomp $line;

      if ($line =~ m/Number of files:\s+(\d+)/) {

         $stats{'files_count'} = $1;

      } elsif ($line =~ m/Number of files transferred:\s+(\d+)/) {

         $stats{'transferred_count'} = $1;

      } elsif ($line =~ m/Total file size:\s+(\d+)\s+bytes/) {

         $stats{'files_size'} = $1;

      } elsif ($line =~ m/Total transferred file size:\s+(\d+)\s+bytes/) {

         $stats{'transferred_size'} = $1;

      } elsif ($line =~ m/sent\s+(\d+)\s+bytes\s+received\s+(\d+)\s+bytes\s+([0-9.]+)\s+bytes\/sec/) {

         $stats{'data_sent'} = $1;

         $stats{'data_received'} = $2;

      } elsif ($line =~ m/^Begin\s+(\d+)\s*\[(\d+)\]/){

         $stats{'start_time'} = qq/'$1'/;

      } elsif ($line =~ m/^Complete\s+(\d+)\s*\[(\d+)\]/){

         $stats{'end_time'} = qq/'$1'/;

      } elsif ($line =~ m/^Duration\s+([\d:]+)\s+\[(\d+) seconds\]/){

         $stats{'Duration'} = $2;

      } elsif ($line =~ m/^(\d+)\[(\d+)\] began/ ) {

         $stats{'start_time'} = qq/'$1'/;

      } elsif ($line =~ m/^(\d+)\[(\d+)\] finished/ ) {

         $stats{'end_time'} = qq/'$1'/;

      } elsif ($line =~ m/Backup Version ([0-9.]+)/) {

         $stats{'version'} = qq/'$1'/;

      } elsif ($line =~ m/Directory Size On Server: ([0-9.]+) bytes/ ) {

         $stats{'StorageSize'} = $1;

      }

   }

   # make sure all columns have a value

   if  ($stats{'start_time'} && $stats{'end_time'} && $stats{'version'}) {

      return qq/insert into backups_run (backups_id,report_date,start_time,end_time,version,

                                          files_count,files_size,transferred_count,

                                          transferred_size,files_deleted,

                                          data_sent,data_received,disk_used)

                           select backups_id, 

                                    '$reportDate',

                                    $stats{'start_time'},

                                    $stats{'end_time'},

                                    $stats{'version'},

                                    $stats{'files_count'},

                                    $stats{'files_size'},

                                    $stats{'transferred_count'},

                                    $stats{'transferred_size'},

                                    $stats{'files_deleted'},

                                    $stats{'data_sent'},

                                    $stats{'data_received'},

                                    $stats{'StorageSize'}

                           from backups

                           where device_id = $deviceID/;

   } else {

      print "Error Processing $filename\n";

      foreach $key ('start_time','end_time','version','files_count','files_size','transferred_count','transferred_size','files_deleted','data_sent','data_received') {

         print "\t$key = [$stats{$key}]\n";

      }

      print "\n";

      return '';

   }

}

sub getAttachments{

   opendir (DATADIR,$sourceDir)  || die "can't opendir $sourceDir: $!"; # read the directory

   #print "Opened $sourceDir\n";

   while ( my $thisFile = readdir(DATADIR) ) { # for each file

      next if $thisFile =~ m/^\./; # skip if it is a "dot" file

      #print "It is not a dot file\n";

      next unless -f "$sourceDir/$thisFile";

      $thisFile = $sourceDir . '/' . $thisFile; # ok, get fully qualified path/filename

      #print "Preparing $thisFile\n";

      `munpack -f -C $tempDir < '$thisFile'`;

      &postProcess($thisFile);

   }

   closedir(DATADIR);

   `rm -f $tempDir/*.desc`;

   `gunzip -f $tempDir/*.gz`;

}

sub getDeviceID {

   my $device = shift;

   $device = &GenericSQL::fixStringValue($dbh, $device);

   my $sql = "select distinct device_id from device where name = $device union select device_id from device_alias where alias = $device";

   #print "\t$sql\n";

   #return 1;

   return &GenericSQL::getOneValue($dbh,$sql);

}

sub parseFileName {

   my $fileName = shift;

   $fileName =~ m/(\d+)_(.+)\.backup\.log/;

   return (&getDeviceID($2),$2,$1);

}

###############################################################################

#            BEGIN MAIN ROUTINE

###############################################################################

BEGIN{

   # load the configuration file

   eval ( &loadConfigurationFile );

   push @INC, $LIBRARIES;

}

use strict;

no strict 'vars';

use Data::Dumper;

use GenericSQL; # generic, home grown MySQL access routines

#use GenericTemplates;

use Logging; # generic, home grown logging routines

use Date::Format; # allows us to format our dates nicely

use Date::Parse; # VERY KEWL, parses out a huge number of date formats

$dbh = DBI->connect( $DSN, $DB_USER , $DB_PASS ) or die $DBI::errstr; # try to connect to db first

&getAttachments; # unpacks all attachments into $tempDir and gunzips them

# now, process each file in turn

opendir (DATADIR,$tempDir)  || die "can't opendir $tempDir: $!"; # read the directory

while ( my $thisFile = readdir(DATADIR) ) { # for each file

   next if $thisFile =~ m/^\./; # skip if it is a "dot" file

   print "It is not a dot file\n" if $TESTING >= 3;

   next unless -f "$tempDir/$thisFile";

   print "Looking at $thisFile\n" if $TESTING >= 2;

   ($device_id,$device,$report_date) = &parseFileName($thisFile);

   print "\tparseFileName = Device: $device_id Date:$report_date\n" if $TESTING >= 2;

   unless ( $device_id ) {

      push @errors, "Could not find device info on $thisFile";

      next;

   }

   unless ( $report_date ) {

      push @errors, "Invalid report date for $thisFile";

      next;

   }

   if ( exists( $backupDomains{ $device } ) ) {

      $backupDomains{ $device } = 1;

   } else {

      push @errors, "$device not created in backups to process, processing anyway";

   } 

   print "\tDevice: $device_id\tDate: $report_date\n" if $TESTING;

   # check for duplicate report

   next if &GenericSQL::getOneValue($dbh,qq/select count(*) from backups_run join backups using (backups_id) where device_id = $device_id and report_date = '$reportDate'/);

   $sql = &processFile($tempDir . '/' . $thisFile, $device_id,$report_date);

   # if no sql returned, it is not a valid file

   unless ( $sql ) {

      push @errors, "Could not generate SQL statement for $thisFile";

      next;

   }

   unlink "$tempDir/$thisFile";

   print "$sql\n" if $TESTING >= 3;

   &GenericSQL::doSQL( $dbh,$sql );

}

closedir(DATADIR);

foreach my $temp ( keys %backupDomains ) {

   push @errors, "No backup for $temp" unless $backupDomains{$temp};

}

if ( @errors ) {

  print "ERRORS DETECTED:\n" . join( "\n", @errors ) . "\n";

  my $message = "Subject: ERRORS on backups\n\n" . join( "\n", @errors ) . "\n";

  open SENDMAIL, "|$SENDMAIL" or die "Can not open $SENDMAIL: $!\n";

  print SENDMAIL "To: $MISSING_BACKUP_MAILTO\n";

  print SENDMAIL $message;

  close SENDMAIL;

}

1;