WebSVN – sysadmin_scripts – Diff – /trunk/archiveDirectories/archiveDirectories.pl

 #! /usr/bin/env perl
 # archiveDirectories.pl
 # Author: R. W. Rodolico
 # Date: 20180603
-# Copyright: 2018, Vanduzen Enterprises, Dallas TX
+# Copyright (c) 2018, Daily Data, Inc
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and documentation are those
+# of the authors and should not be interpreted as representing official policies,
+# either expressed or implied, of the <project name> project.
 # Script designed to be run from a cron job, which checks if any directories
 # are ready to be archived. A directory is defined as a directory under
 # the root of $config{'local root dir'}.
 # $config{'target final directory'}.
 # After the copy and move, the directory and its MD5 sum file are moved
 # to the $config{'local trash dir'} (which is cleaned on the next invocation of
 # the script).
-# Script does NOT handle the situation where directories are being moved
-# while the script is running, so the script should be run at a time
-# when there is no other activity on the server.
+#
 # Version: 1.0
 use warnings;
 use strict;
 use Cwd qw();
 use File::Copy qw(move);
 use File::Basename;
 use File::stat;
+our $VERSION = '1.0';
-my $DEBUG = 5;
+my $DEBUG = 0;
 my %config;
 my @DirectoriesToMove;
    } else {
       die "Could not locate config file $configFileName\n";
    } # if..else
 } #loadConfig
-# simply read the entire fiel into a string
+# simply read the entire file into a string
 sub slurpFile {
    my $filename = shift;
    return '' unless -e $filename;
    open TEMP, "<$filename" or die "could not read $filename: $!\n";
    my @contents = <TEMP>;
    open TEMP, ">$filename" or die "could not write to $filename: $!\n";
    print TEMP join( '', @_ );
    close TEMP;
+}
+# returns how many seconds ago a file was created
+sub fileAge {
+   my $filename = shift;
+   my $age = stat( $filename );
+   $age = $$age[9];
+   print "$age\t$filename" if $DEBUG > 3;
+   return time - $age;
+}
 # look in the directories to move directory and see if there is anything
 # new in there. If so, check MD5 Sum file (create if necessary) and ensure
 # we have waited long enough and the sums match
 sub getDirectories {
    my $rootDir = shift;
       print "\tFound Dir $fullyQualified with MD5 of $md5\n" if $DEBUG > 2;
       # let's look for the md5 checksum file and compare if it exist
       my $md5Name = "$fullyQualified.$config{'md5 suffix'}";
       if ( -e $md5Name ) {
          # find out when it was last written to
-         my $lastModification = stat( $md5Name );
-         $lastModification = $$lastModification[9];
-         my $howOld = time - $lastModification;
-         print "\tFound existing MD5 file $md5Name written to at $lastModification, or $howOld seconds ago\n" if $DEBUG > 3;
+         print "\tFound existing MD5 file $md5Name\n" if $DEBUG > 3;
          # and blow it off if it is too recent
-         if ( $howOld < $config{'quiesent seconds'} ) {
+         if ( &fileAge( $md5Name) < $config{'quiesent seconds'} ) {
-            print "\t\tBlowing it off because $howOld is less than $config{'quiesent seconds'}\n" if $DEBUG > 4;
+            print "\t\tBlowing it off because it is less than $config{'quiesent seconds'} seconds old\n" if $DEBUG > 4;
             next;
+         }
          my $oldMD5 = &slurpFile( $md5Name );
          if ( $md5 eq $oldMD5 ) {
             print "\t\tAdding, md5 not changed, $md5 same as $oldMD5\n" if $DEBUG > 4;
+   }
    return ('', 0);
+}
-# simply remove everything from the trash directory
-sub cleanTrash {
-   my ( $trashDir, $age ) = @_;
-   `mkdir -p $trashDir` unless -d $trashDir;
-   `rm -fR $trashDir/*`;
-}
 sub copyToRemote {
    my ( $path, $dirname, $remoteServer, $remotePath ) = @_;
    # first, copy the file
    #print "rsync -a $path/$dirname $remoteServer:$remotePath > /tmp/lastrsync.log";
    #die;
    qx"rsync -a $path/$dirname $remoteServer:$remotePath > /tmp/lastrsync.log";
    return 'rsync failed with error :' . $? & 127 if $? & 127;
    return '';
+}
+# simply remove everything from the trash directory over $age seconds old
+sub cleanTrash {
+   my ( $trashDir, $age ) = @_;
+   my $md5Suffix = $config{'md5 suffix'};
+   my @toRemove = ();
+   if ( opendir( my $dh, $trashDir ) ) {
+      # get all the md5sum files which are older than $age seconds old
+      @toRemove = grep { &fileAge( "$_" ) > $age  } map{ "$trashDir/$_" }  grep{ /$md5Suffix$/ } readdir( $dh);
+      closedir( $dh );
+   }
+   print "You should remove the following files\n" if $DEBUG > 1;
+   foreach my $thisDir ( @toRemove ) {
+      $thisDir =~ m/(.*)\.$md5Suffix/;
+      $thisDir = $1;
+      qx/rm -fR $thisDir $thisDir.$md5Suffix/;
+   }
+}
 ###############################################################################
 # Main
 ###############################################################################
 &loadConfig();
    `mkdir -p $config{'local root dir'}`;
    `chmod 777 $config{'local root dir'}`;
+}
 # clean the trash if $config{ 'trash cleanup' } is non-zero
 &cleanTrash( $config{'local trash dir'}, $config{ 'trash cleanup' } ) if $config{ 'trash cleanup' };
 # Check if we have any directories which are ready to be moved.
 @DirectoriesToMove = &getDirectories( $config{'local root dir'} );
-print "Processing\n\t" . join( "\n\t", @DirectoriesToMove ) . "\n";
+print "Processing\n\t" . join( "\n\t", @DirectoriesToMove ) . "\n" if $DEBUG > 1;
 foreach my $directory ( @DirectoriesToMove ) {
    my $fullPath = $config{'local root dir'} . "/$directory";
    my $logFile = "$fullPath.$config{'log suffix'}";
    my $errorFile = "$fullPath.$config{'error suffix'}";
 opendir( my $dh, $config{'local staging area'} ) or die "Could not read $config{'local staging area'}: $!\n";
 my @directories;
 # get all the .md5 files
 my @toMove = grep { /$config{'md5 suffix'}$/ } readdir( $dh );
 my $targetPath = "$config{'target server'}:$config{'target staging area'}/";
-print "Copying the following to $targetPath\n\t" . join ("\n\t", @toMove ) . "\n";
+print "Copying the following to $targetPath\n\t" . join ("\n\t", @toMove ) . "\n" if $DEBUG > 1;
 # create the target directory on the server if it doesn't exist
 &runRemoteCommand( $config{'target server'},
    "[ ! -d $config{'target staging area'} ] && mkdir -p $config{'target staging area'}",
    "[ ! -d $config{'target final directory'} ] && mkdir -p $config{'target final directory'}"
    );
    &logit( $dirname, $config{'log suffix'}, $rsync );
    if ( system ( $rsync ) == 0 ) { # we succeeded
       if ( &validateTarget( $config{'target server'}, $config{'target staging area'}, $config{'target final directory'}, $dirname, $md5sum ) ) {
          `mkdir -p $config{'local trash dir'}` unless -d $config{'local trash dir'};
          move( "$config{'local staging area'}/$dirname", "$config{'local trash dir'}/$dirname" );
-         $dirname .= $config{'md5 suffix'};
+         my $md5File = $dirname . '.' . $config{'md5 suffix'};
-         move( "$config{'local staging area'}/$dirname", "$config{'local trash dir'}/$dirname" );
+         move( "$config{'local staging area'}/$md5File", "$config{'local trash dir'}/$md5File" );
          &logit( $dirname, $config{'log suffix'}, "Successfully moved directory $dirname to $config{'target server'}" );
       } else {
          &logit( $dirname, $config{'error suffix'}, "Unable to validate target for $dirname" );
+      }
    } else {

Subversion Repositories sysadmin_scripts

(root)/trunk/archiveDirectories/archiveDirectories.pl – Rev 17 → 18