WebSVN – sysadmin_scripts – Diff – /trunk/archiveProjects/archiveDirectories.pl

 #! /usr/bin/env perl
-# archiveDirectorys.pl
+# archiveDirectories.pl
 # Author: R. W. Rodolico
 # Date: 20180603
 # Copyright: 2018, Vanduzen Enterprises, Dallas TX
-# Script designed to be run from a cron job, which checks if any directorys
+# Script designed to be run from a cron job, which checks if any directories
 # are ready to be archived. A directory is defined as a directory under
-# the root of $localDirectoryDirectory.
+# the root of $config{'local root dir'}.
 # If found, all directories are moved into the staging area and
 # an md5 checksum is calculated for the entire tree.
-# After all directorys are moved, a second process looks in the staging
+# After all directories are moved, a second process looks in the staging
 # area and copies the files (using rsync for reliability) into the staging
-# area of $targetServer. When a directory has been copied, a checksum is
+# area of $config{'target server'}. When a directory has been copied, a checksum is
 # calculated on the remote copy and compared to the checksum calculated
 # in the first stage and, if it passes, the directory is then moved to the
-# $targetDirectoryDirectory.
+# $config{'target final directory'}.
 # After the copy and move, the directory and its MD5 sum file are moved
-# to the $trashDirectory (which is cleaned on the next invocation of
+# to the $config{'local trash dir'} (which is cleaned on the next invocation of
 # the script).
-# Script does NOT handle the situation where directorys are being moved
+# Script does NOT handle the situation where directories are being moved
 # while the script is running, so the script should be run at a time
 # when there is no other activity on the server.
+#
 # Version: 1.0
 use warnings;
 use strict;
 use Cwd qw();
 use File::Copy qw(move);
 use File::Basename;
+use File::stat;
+my $DEBUG = 5;
+my %config = (
-# location where directorys are put by end users
+   # location where directories are put by end users
-my $localDirectoryDirectory = '/home/samba/transfers/denver_to_dallas/Archive_to_DaVinci';
+   'local root dir' => '/home/rodolico/scripts/sysadmin_scripts/archiveProjects/temp/ArchiveProjects',
-# location where directories are moved while processing
+   # location where directories are moved while processing
-my $rootWorkDirectory = '/home/transfer_work_area';
+   'local work dir' => '/home/rodolico/scripts/sysadmin_scripts/archiveProjects/temp/transfer_area',
-# location where directories are moved when job is completed
+   # location where directories are moved when job is completed
-my $trashDirectory = "$localDirectoryDirectory/.Trash";
+   'local trash dir' => "/home/rodolico/scripts/sysadmin_scripts/archiveProjects/temp/Trash",
-# location where directories are moved while being transferred
+   # location where directories are moved while being transferred
-my $stagingArea = "$localDirectoryDirectory/.Staging";
+   'local staging area' => '/home/rodolico/scripts/sysadmin_scripts/archiveProjects/temp/Staging',
-# target server name/ip. Must be accessible via ssh with no password
+   # target server name/ip. Must be accessible via ssh with no password
-my $targetServer = 'davinci';
+   'target server' => 'davinci',
-# location on target server where directories are placed while copying
+   # location on target server where directories are placed while copying
-my $targetStagingArea = '/home/samba/archives/fromDenver/.Staging/';
+   'target staging area' => '/home/samba/archives/fromDenver/.Staging/',
-# location on target server where directories are finally put
+   # location on target server where directories are finally put
-my $targetDirectoryDirectory = '/home/samba/archives/fromDenver/';
+   'target final directory' => '/home/samba/archives/fromDenver/',
-# suffix of md5 of directories
+   # suffix of md5 of directories
-my $md5suffix = '.md5sum';
+   'md5 suffix' => 'md5sum',
+   # suffix of filename to create showing actions
+   'log suffix' => 'log',
+   # suffix of error log
+   'error suffix' => 'err',
+   # how long a directory must be undisturbed before it is ready to work on
+   'quiesent seconds' => 60*5, # five minutes
+   # how long to leave stuff in the trash directory. 0 indicates never do it.
+   'trash cleanup' => 86400*7, # 7 days
+);
-my @DirectorysToMove;
+my @DirectoriesToMove;
+# simply read the entire fiel into a string
+sub slurpFile {
+   my $filename = shift;
+   return '' unless -e $filename;
+   open TEMP, "<$filename" or die "could not read $filename: $!\n";
+   my @contents = <TEMP>;
+   close TEMP;
+   return join( '', @contents );
+}
+# print a value to a file
+sub writeData {
+   my $filename = shift;
+   open TEMP, ">$filename" or die "could not write to $filename: $!\n";
+   print TEMP join( '', @_ );
+   close TEMP;
+}
-# look in the directorys to move directory and see if there is anything
+# look in the directories to move directory and see if there is anything
+# new in there. If so, check MD5 Sum file (create if necessary) and ensure
-# new in there.
+# we have waited long enough and the sums match
-sub getDirectorys {
+sub getDirectories {
    my $rootDir = shift;
+   print "In getDirectories with dir of $rootDir\n" if $DEBUG;
    opendir( my $dh, $rootDir ) or die "Could not open directory $rootDir: $!\n";
    my @dirs = grep { ! /^\./ && -d "$rootDir/$_" } readdir( $dh );
+   closedir ( $dh );
+   print "Directories Found\n" . join( "\n", @dirs ) . "\n" if $DEBUG > 1;
+   my @dirsToMove;
+   foreach my $thisDir ( @dirs ) {
+      my $fullyQualified = "$rootDir/$thisDir";
+      my $md5 = calcMD5( $fullyQualified );
+      print "\tFound Dir $fullyQualified with MD5 of $md5\n" if $DEBUG > 2;
+      # let's look for the md5 checksum file and compare if it exist
+      my $md5Name = "$fullyQualified.$config{'md5 suffix'}";
+      if ( -e $md5Name ) {
+         # find out when it was last written to
+         my $lastModification = stat( $md5Name );
+         $lastModification = $$lastModification[9];
+         my $howOld = time - $lastModification;
+         print "\tFound existing MD5 file $md5Name written to at $lastModification, or $howOld seconds ago\n" if $DEBUG > 3;
+         # and blow it off if it is too recent
+         if ( $howOld < $config{'quiesent seconds'} ) {
+            print "\t\tBlowing it off because $howOld is less than $config{'quiesent seconds'}\n" if $DEBUG > 4;
+            next;
+         }
+         my $oldMD5 = &slurpFile( $md5Name );
+         if ( $md5 eq $oldMD5 ) {
+            print "\t\tAdding, md5 not changed, $md5 same as $oldMD5\n" if $DEBUG > 4;
+            push @dirsToMove, $thisDir;
+         } else {
+            print "\t\tWaiting, md5 changed, $md5 and $oldMD5\n" if $DEBUG > 4;
+            # overwrite if the checksum has changed
+            &writeData( $md5Name, $md5 ) if $md5 ne &slurpFile( $md5Name );
+         }
+      } else { # doesn't exist, so create it
+         print "\t\tCreating MD5 File $md5Name with value $md5\n" if $DEBUG > 4;
+         &writeData( $md5Name, $md5 );
+      }
+   } # foreach
-   return @dirs;
+   return @dirsToMove;
+}
 # calculate the checksum of a directory by
 # 1. calculate checksum of each individual file in the entire tree
 # 2. Grab the first column, which is the checksum
+}
 # moves directory to staging area and puts the md5 sum into a file
 # with the same name, but a .md5sum suffix
 sub moveToStaging {
-   my ( $directory, $stagingArea, $md5 ) = @_;
+   my ( $directory, $fullPath, $staging ) = @_;
+   # and let's get the md5 file name also
+   my $md5File = $fullPath . ".$config{'md5 suffix'}";
-   mkdir $stagingArea unless -d $stagingArea;
+   mkdir $staging unless -d $staging;
-   move( "$localDirectoryDirectory/$directory", "$stagingArea/$directory" );
+   return 'Directory already exists in staging' if -e "$staging/$directory";
-   my $md5File = "$stagingArea/$directory" . $md5suffix;
+   move( $fullPath, "$staging/$directory" ) or die "Error moving $fullPath to $staging/$directory: $!\n";
-   open DATA,">$md5File" or die "Could not create md5sum file [$md5File]: $!\n";
+   move( $md5File, $staging ) or die "Error moving $md5File to $staging: $!\n";
-   print DATA "$md5\n";
-   close DATA;
-   return;
+   return '';
+}
 # verifies the directory is correct on the server by comparing the checksums
 # calculated locally and remote server. If valid, moves it into the final
 # location on the remote server
+}
 # reads the checksum file
 sub getCheckSum {
    my ( $directory, $staging )  = @_;
-   $directory .= $md5suffix;
+   $directory .= $config{'md5 suffix'};
    if ( open DATA, "<$staging/$directory" ) {
       my $cksum = <DATA>;
       chomp $cksum;
       close DATA;
       return $cksum;
    return '';
+}
 # simple little logger that records some information
 sub logit {
-   my $message = shift;
+   my $logfile = shift;
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
    my $now = sprintf( "%04d-%02d-%02d %02d:%-2d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec );
-   open LOG, ">>/tmp/archiveDirectorys.log" or die "could not write to archiveDirectorys.log: $!\n";
+   open LOG, ">>$logfile" or die "could not write to $logfile: $!\n";
+   while ( my $message = shift ) {
-   print LOG "$now\t$message\n";
+      print LOG "$now\t$message\n";
+   }
    close LOG;
+}
 # simply remove everything from the trash directory
 sub cleanTrash {
-   my $trashDir = shift;
+   my ( $trashDir, $age ) = @_;
+   `mkdir -p $trashDir` unless -d $trashDir;
    `rm -fR $trashDir/*`;
+}
+unless ( -d $config{'local root dir'} ) {
+   `mkdir -p $config{'local root dir'}`;
+   `chmod 777 $config{'local root dir'}`;
+}
+# clean the trash if $config{ 'trash cleanup' } is non-zero
-#&cleanTrash( $trashDirectory ) if &getDirectorysToMove( $trashDirectory );
+&cleanTrash( $config{'local trash dir'}, $config{ 'trash cleanup' } ) if $config{ 'trash cleanup' };
-# first, check and see if we have any directorys we need to move
+# Check if we have any directories which are ready to be moved.
-@DirectorysToMove = &getDirectorys( $localDirectoryDirectory );
+@DirectoriesToMove = &getDirectories( $config{'local root dir'} );
+print "Processing\n\t" . join( "\n\t", @DirectoriesToMove ) . "\n";
-foreach my $directory ( @DirectorysToMove ) {
+foreach my $directory ( @DirectoriesToMove ) {
-   my $md5 = &calcMD5( "$localDirectoryDirectory/$directory" );
+   my $fullPath = $config{'local root dir'} . "/$directory";
+   my $logFile = "$fullPath.$config{'log suffix'}";
+   my $errorFile = "$fullPath.$config{'error suffix'}";
+   print "Path for $directory is $fullPath\n\tLog File is $logFile\n\tError file is $errorFile\n" if $DEBUG > 3;
+   if ( -e $errorFile ) {
+      &logit( $logFile, "Aborting because we have a pre-existing error" );
+      print "\tAborting because we have a pre-existing error\n" if $DEBUG > 3;
+      next;
+   }
-   &logit( "New Directory $md5\t$directory" );
+   &logit( $logFile, "Processing $directory" );
-   &moveToStaging( $directory, $stagingArea, $md5 );
+   my $error = &moveToStaging( $directory, $fullPath, $config{'local staging area'} );
+   if ( ! $error ) {
+      print "\tMoved to $config{'local staging area'}\n" if $DEBUG > 3;
+      &logit( $logFile, "Successfully moved to $config{'local staging area'}" );
+   } else {
+      &logit( $logFile, "Error, move aborted" );
+      &logit( $errorFile, $error );
+   }
+}
 # done with that, now we need to see if there is anything in the staging area
 # that needs to be sent to the remote server
-opendir( my $dh, $stagingArea ) or die "Could not read $stagingArea: $!\n";
+opendir( my $dh, $config{'local staging area'} ) or die "Could not read $config{'local staging area'}: $!\n";
 my @directories;
-my @toMove = grep { /$md5suffix$/ } readdir( $dh );
+my @toMove = grep { /$config{'md5 suffix'}$/ } readdir( $dh );
+my $targetPath = "$config{'target server'}:$config{'target staging area'}/";
+print "Copying the following to $targetPath\n\t" . join ("\n\t", @toMove ) . "\n";
+die;
 foreach my $directory ( @toMove ) {
    $directory =~ m/^(.*)\.md5sum/;
    $directory = $1;
-   my $md5sum = &getCheckSum( $directory, $stagingArea );
+   my $md5sum = &getCheckSum( $directory, $config{'local staging area'} );
    next unless $md5sum;
-   my $rsync = "rsync -av '$stagingArea/$directory' $targetServer:$targetStagingArea/ > /tmp/lastrsync.log";
+   my $rsync = "rsync -av '$config{'local staging area'}/$directory' $config{'target server'}:$config{'target staging area'}/ > /tmp/lastrsync.log";
    &logit( $rsync );
    if ( system ( $rsync ) == 0 ) { # we succeeded
-      if ( &validateTarget( $targetServer, $targetStagingArea, $targetDirectoryDirectory, $directory, $md5sum ) ) {
+      if ( &validateTarget( $config{'target server'}, $config{'target staging area'}, $config{'target final directory'}, $directory, $md5sum ) ) {
-         `mkdir -p $trashDirectory` unless -d $trashDirectory;
+         `mkdir -p $config{'local trash dir'}` unless -d $config{'local trash dir'};
-         move( "$stagingArea/$directory", "$trashDirectory/$directory" );
+         move( "$config{'local staging area'}/$directory", "$config{'local trash dir'}/$directory" );
-         $directory .= $md5suffix;
+         $directory .= $config{'md5 suffix'};
-         move( "$stagingArea/$directory", "$trashDirectory/$directory" );
+         move( "$config{'local staging area'}/$directory", "$config{'local trash dir'}/$directory" );
-         &logit( "Successfully moved directory $directory to $targetServer" );
+         &logit( "Successfully moved directory $directory to $config{'target server'}" );
       } else {
          &logit( "Unable to validate target for $directory" );
+      }
    } else {
       &logit( "Unknown error attempting to rsync $directory" );

Subversion Repositories sysadmin_scripts

(root)/trunk/archiveProjects/archiveDirectories.pl @ 13 – Rev 12 → 13