Subversion Repositories sysadmin_scripts

#! /usr/bin/env perl

# archiveDirectories.pl

# Author: R. W. Rodolico

# Date: 20180603

# Copyright (c) 2018, Daily Data, Inc

# All rights reserved.

# 

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are met:

# 

# 1. Redistributions of source code must retain the above copyright notice, this

#    list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright notice,

#    this list of conditions and the following disclaimer in the documentation

#    and/or other materials provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES

# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS

# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and documentation are those

# of the authors and should not be interpreted as representing official policies,

# either expressed or implied, of the <project name> project.

# Script designed to be run from a cron job, which checks if any directories

# are ready to be archived. A directory is defined as a directory under

# the root of $config{'local root dir'}.

# If found, all directories are moved into the staging area and 

# an md5 checksum is calculated for the entire tree.

# After all directories are moved, a second process looks in the staging

# area and copies the files (using rsync for reliability) into the staging

# area of $config{'target server'}. When a directory has been copied, a checksum is

# calculated on the remote copy and compared to the checksum calculated

# in the first stage and, if it passes, the directory is then moved to the 

# $config{'target final directory'}.

# After the copy and move, the directory and its MD5 sum file are moved

# to the $config{'local trash dir'} (which is cleaned on the next invocation of

# the script).

#

# Version: 1.0

use warnings;

use strict;

use Cwd qw();

use File::Copy qw(move);

use File::Basename;

use File::stat;

our $VERSION = '1.0';

my $DEBUG = 3;

my %config;

my @DirectoriesToMove;

sub loadConfig {

   use FindBin;

   my $configFileName = "$FindBin::Bin/$FindBin::Script";

   unless ( $configFileName =~ s/\.pl$/\.conf/ ) {

      $configFileName .= '.conf';

   }

   if ( -e $configFileName ) {

      my $configFileContents = &slurpFile( $configFileName );

      eval( $configFileContents );

      die "Error interpreting $configFileName: $@\n" if $@;

   } else {

      die "Could not locate config file $configFileName\n";

   } # if..else

} #loadConfig

# simply read the entire file into a string

sub slurpFile {

   my $filename = shift;

   return '' unless -e $filename;

   open TEMP, "<$filename" or die "could not read $filename: $!\n";

   my @contents = <TEMP>;

   close TEMP;

   return join( '', @contents );

}

# print a value to a file

sub writeData {

   my $filename = shift;

   open TEMP, ">$filename" or die "could not write to $filename: $!\n";

   print TEMP join( '', @_ );

   close TEMP;

}

# returns how many seconds ago a file was created

sub fileAge {

   my $filename = shift;

   my $age = stat( $filename );

   $age = $$age[9];

   print "$age\t$filename" if $DEBUG > 3;

   return time - $age;

}

# look in the directories to move directory and see if there is anything 

# new in there. If so, check MD5 Sum file (create if necessary) and ensure

# we have waited long enough and the sums match

sub getDirectories {

   my $rootDir = shift;

   print "In getDirectories with dir of $rootDir\n" if $DEBUG;

   opendir( my $dh, $rootDir ) or die "Could not open directory $rootDir: $!\n";

   my @dirs = grep { ! /^\./ && -d "$rootDir/$_" } readdir( $dh );

   closedir ( $dh );

   print "Directories Found\n" . join( "\n", @dirs ) . "\n" if $DEBUG > 1;

   my @dirsToMove;

   foreach my $thisDir ( @dirs ) {

      my $fullyQualified = "$rootDir/$thisDir";

      my $md5 = calcMD5( $fullyQualified );

      print "\tFound Dir $fullyQualified with MD5 of $md5\n" if $DEBUG > 2;

      # let's look for the md5 checksum file and compare if it exist

      my $md5Name = "$fullyQualified.$config{'md5 suffix'}";

      if ( -e $md5Name ) {

         # find out when it was last written to

         print "\tFound existing MD5 file $md5Name\n" if $DEBUG > 3;

         # and blow it off if it is too recent

         if ( &fileAge( $md5Name) < $config{'quiesent seconds'} ) {

            print "\t\tBlowing it off because it is less than $config{'quiesent seconds'} seconds old\n" if $DEBUG > 4;

            next;

         }

         my $oldMD5 = &slurpFile( $md5Name );

         if ( $md5 eq $oldMD5 ) {

            print "\t\tAdding, md5 not changed, $md5 same as $oldMD5\n" if $DEBUG > 4;

            push @dirsToMove, $thisDir;

         } else {

            print "\t\tWaiting, md5 changed, $md5 and $oldMD5\n" if $DEBUG > 4;

            # overwrite if the checksum has changed

            &writeData( $md5Name, $md5 ) if $md5 ne &slurpFile( $md5Name );

         }

      } else { # doesn't exist, so create it

         print "\t\tCreating MD5 File $md5Name with value $md5\n" if $DEBUG > 4;

         &writeData( $md5Name, $md5 );

      }

   } # foreach

   return @dirsToMove;

}

# calculate the checksum of a directory by

# 1. calculate checksum of each individual file in the entire tree

# 2. Grab the first column, which is the checksum

# 3. sort the result since Linux will not always return them in the same order

# 4. do a checksum of the checksums

#

# This is highly unlikely to give the same answer if any file changes

# in the process of the copy

sub calcMD5 {

   my $directory = shift;

   return -1 unless -d $directory;

   my $md5 = `find '$directory' -type f -exec md5sum \\{\\} \\; | cut -d' ' -f1 | sort | md5sum | cut -d' ' -f1`;

   chomp $md5;

   return $md5;

}

# moves directory to staging area and puts the md5 sum into a file

# with the same name, but a .md5sum suffix

sub moveToStaging {

   my ( $directory, $fullPath, $staging ) = @_;

   # and let's get the md5 file name also

   my $md5File = $fullPath . ".$config{'md5 suffix'}";

   mkdir( $staging ) unless -d $staging;

   return 'Directory already exists in staging' if -e "$staging/$directory";

   move( $fullPath, "$staging/$directory" ) or die "Error moving $fullPath to $staging/$directory: $!\n";

   move( $md5File, $staging ) or die "Error moving $md5File to $staging: $!\n";

   return '';

}

sub getCheckSum {

   my $project = shift;

   my $checkSumFile = $config{'local staging area'} . '/' . $project . '.' . $config{'md5 suffix'};

   if ( -e $checkSumFile ) {

      return &slurpFile( $checkSumFile );

   }

   return '';

}

# verifies the directory is correct on the server by comparing the checksums

# calculated locally and remote server. If valid, moves it into the final

# location on the remote server

sub validateTarget {

   my ( $remoteServer, $remoteStaging, $remoteTarget, $directory, $checksum ) = @_;

   my $md5sum = `ssh $remoteServer "find '$remoteStaging/$directory' -type f -exec md5sum \\{\\} \\; | cut -d' ' -f1 | sort | md5sum | cut -d' ' -f1"`;

   chomp $md5sum;

   if ( $checksum eq $md5sum ) {

      my $command = "ssh $remoteServer \"mv '$remoteStaging/$directory' '$remoteTarget'\"";

      if ( system( $command ) == 0 ) {

         return 1;

      } else {

         &logit( "Unable to move $directory to $remoteServer:$remoteTarget" );

         return 0;

      }

   } else {

      &logit( "Invalid checksum moving directory $directory" );

      return 0;

   }

}

# simple little logger that records some information   

sub logit {

   my $projectName = shift;

   my $suffix = shift;

   my $logfile = $config{'local root dir'} . "/$projectName.$suffix";

   my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);

   my $now = sprintf( "%04d-%02d-%02d %02d:%-2d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec );

   open LOG, ">>$logfile" or die "could not write to $logfile: $!\n";

   while ( my $message = shift ) {

      print LOG "$now\t$message\n";

   }

   close LOG;

}

sub runRemoteCommand {

   my $server = shift;

   while ( my $command = shift ) {

      my $output = qx/ssh $server '$command'/;

      if ( my $error = $? & 127 ) {

         return ( $output, $error );

      }

   }

   return ('', 0);

}

sub copyToRemote {

   my ( $path, $dirname, $remoteServer, $remotePath ) = @_;

   # first, copy the file

   #print "rsync -a $path/$dirname $remoteServer:$remotePath > /tmp/lastrsync.log";

   #die;

   qx"rsync -a $path/$dirname $remoteServer:$remotePath > /tmp/lastrsync.log";

   return 'rsync failed with error :' . $? & 127 if $? & 127;

   return '';

}

# simply remove everything from the trash directory over $age seconds old

sub cleanTrash {

   my ( $trashDir, $age ) = @_;

   my $md5Suffix = $config{'md5 suffix'};

   my @toRemove = ();

   if ( opendir( my $dh, $trashDir ) ) {

      # get all the md5sum files which are older than $age seconds old

      @toRemove = grep { &fileAge( "$_" ) > $age  } map{ "$trashDir/$_" }  grep{ /$md5Suffix$/ } readdir( $dh);

      closedir( $dh );

   }

   print "You should remove the following files\n" if $DEBUG > 1;

   foreach my $thisDir ( @toRemove ) {

      $thisDir =~ m/(.*)\.$md5Suffix/;

      $thisDir = $1;

      qx/rm -fR '$thisDir' '$thisDir.$md5Suffix'/;

   }

}

sub makeDirectories {

   my $directory = shift;

   my $permissions = shift;

   $permissions = '777' unless $permissions;

   unless ( -d $directory ) {

      print "Making directory $directory\n" if $DEBUG > 1;

      `mkdir -p $directory`; 

      `chmod $permissions $directory`;

   }

}

###############################################################################

# Main

###############################################################################

&loadConfig();

#use Data::Dumper;

#print Dumper( \%config );

#die;

foreach my $dirsToMake ( 'local root dir', 'local trash dir', 'local staging area' ) {

   &makeDirectories( $config{$dirsToMake} );

}

# clean the trash if $config{ 'trash cleanup' } is non-zero

&cleanTrash( $config{'local trash dir'}, $config{ 'trash cleanup' } ) if $config{ 'trash cleanup' };

# Check if we have any directories which are ready to be moved.

@DirectoriesToMove = &getDirectories( $config{'local root dir'} );

print "Processing\n\t" . join( "\n\t", @DirectoriesToMove ) . "\n" if $DEBUG > 1;

foreach my $directory ( @DirectoriesToMove ) {

   my $fullPath = $config{'local root dir'} . "/$directory";

   my $logFile = "$fullPath.$config{'log suffix'}";

   my $errorFile = "$fullPath.$config{'error suffix'}";

   print "Path for $directory is $fullPath\n\tLog File is $logFile\n\tError file is $errorFile\n" if $DEBUG > 3;

   if ( -e $errorFile ) {

      &logit( $directory, $config{'log suffix'}, "Aborting because we have a pre-existing error" );

      print "\tAborting because we have a pre-existing error\n" if $DEBUG > 3;

      next;

   }

   &logit( $directory, $config{'log suffix'}, "Processing $directory" );

   my $error = &moveToStaging( $directory, $fullPath, $config{'local staging area'} );

   if ( ! $error ) {

      print "\tMoved to $config{'local staging area'}\n" if $DEBUG > 3;

      &logit( $directory, $config{'log suffix'},  "Successfully moved to $config{'local staging area'}" );

   } else {

      &logit( $directory, $config{'log suffix'},  "Error, move aborted" );

      &logit( $directory, $config{'error suffix'},  $error );

   }

}

# done with that, now we need to see if there is anything in the staging area

# that needs to be sent to the remote server

`mkdir -p $config{'local staging area'}` unless -d $config{'local staging area'};

opendir( my $dh, $config{'local staging area'} ) or die "Could not read $config{'local staging area'}: $!\n";

my @directories;

# get all the .md5 files

my @toMove = grep { /$config{'md5 suffix'}$/ } readdir( $dh );

my $targetPath = "$config{'target server'}:$config{'target staging area'}/";

print "Copying the following to $targetPath\n\t" . join ("\n\t", @toMove ) . "\n" if $DEBUG > 1;

# create the target directory on the server if it doesn't exist

&runRemoteCommand( $config{'target server'},

   "[ ! -d $config{'target staging area'} ] && mkdir -p $config{'target staging area'}",

   "[ ! -d $config{'target final directory'} ] && mkdir -p $config{'target final directory'}"

   );

# now, process each directory in turn

foreach my $dirname ( @toMove ) {

   print "Processing $dirname\n";

   my $error;

   $dirname =~ m/^(.*)\.$config{'md5 suffix'}$/;

   $dirname = $1;

   $error = &copyToRemote( $config{'local staging area'}, $dirname, $config{'target server'}, $config{'target staging area'} );

   if ( $error ) {

      &logit( $dirname, $config{'error suffix'}, $error );

      next;

   } else {

      &logit( $dirname, $config{'log suffix'}, "Copied to $config{'target server'}:$config{'target staging area'}" );

   }

   my $md5sum = &getCheckSum( $dirname );

   next unless $md5sum;

   my $rsync = "rsync -av '$config{'local staging area'}/$dirname' $config{'target server'}:$config{'target staging area'}/ > /tmp/lastrsync.log";

   &logit( $dirname, $config{'log suffix'}, $rsync );

   if ( system ( $rsync ) == 0 ) { # we succeeded

      if ( &validateTarget( $config{'target server'}, $config{'target staging area'}, $config{'target final directory'}, $dirname, $md5sum ) ) {

         `mkdir -p $config{'local trash dir'}` unless -d $config{'local trash dir'};

         move( "$config{'local staging area'}/$dirname", "$config{'local trash dir'}/$dirname" );

         my $md5File = $dirname . '.' . $config{'md5 suffix'};

         move( "$config{'local staging area'}/$md5File", "$config{'local trash dir'}/$md5File" );

         &logit( $dirname, $config{'log suffix'}, "Successfully moved directory $dirname to $config{'target server'}" );

      } else {

         &logit( $dirname, $config{'error suffix'}, "Unable to validate target for $dirname" );

      }

   } else {

      &logit( $dirname, $config{'error suffix'}, "Unknown error attempting to rsync $dirname" );

   }

}

1;