Subversion Repositories sysadmin_scripts

#! /usr/bin/env perl

# archvieProjects.pl

# Author: R. W. Rodolico

# Date: 20180603

# Copyright: 2018, Vanduzen Enterprises, Dallas TX

# Script designed to be run from a cron job, which checks if any projects

# are ready to be archived. A project is defined as a directory under

# the root of $localProjectDirectory.

# If found, all directories are moved into the staging area and 

# an md5 checksum is calculated for the entire tree.

# After all projects are moved, a second process looks in the staging

# area and copies the files (using rsync for reliability) into the staging

# area of $targetServer. When a project has been copied, a checksum is

# calculated on the remote copy and compared to the checksum calculated

# in the first stage and, if it passes, the project is then moved to the 

# $targetProjectDirectory.

# After the copy and move, the project and its MD5 sum file are moved

# to the $trashDirectory (which is cleaned on the next invocation of

# the script).

# Script does NOT handle the situation where projects are being moved

# while the script is running, so the script should be run at a time

# when there is no other activity on the server.

#

# Version: 1.0

use warnings;

use strict;

use Cwd qw();

use File::Copy qw(move);

use File::Basename;

my $localProjectDirectory = '/home/samba/transfers/denver_to_dallas/Archive_to_DaVinci';

my $trashDirectory = "$localProjectDirectory/.Trash";

my $stagingArea = "$localProjectDirectory/.Staging";

my $targetProjectDirectory = '/home/samba/archives/fromDenver/';

my $targetStagingArea = '/home/samba/archives/fromDenver/.Staging/';

my $targetServer = 'davinci';

my $md5suffix = '.md5sum';

my @ProjectsToMove;

# look in the projects to move directory and see if there is anything 

# new in there.

sub getProjects {

   my $rootDir = shift;

   opendir( my $dh, $rootDir ) or die "Could not open directory $rootDir: $!\n";

   my @dirs = grep { ! /^\./ && -d "$rootDir/$_" } readdir( $dh );

   return @dirs;

}

# calculate the checksum of a directory by

# 1. calculate checksum of each individual file in the entire tree

# 2. Grab the first column, which is the checksum

# 3. sort the result since Linux will not always return them in the same order

# 4. do a checksum of the checksums

#

# This is highly unlikely to give the same answer if any file changes

# in the process of the copy

sub calcMD5 {

   my $directory = shift;

   return -1 unless -d $directory;

   my $md5 = `find '$directory' -type f -exec md5sum \\{\\} \\; | cut -d' ' -f1 | sort | md5sum | cut -d' ' -f1`;

   chomp $md5;

   return $md5;

}

# moves project to staging area and puts the md5 sum into a file

# with the same name, but a .md5sum suffix

sub moveToStaging {

   my ( $project, $stagingArea, $md5 ) = @_;

   mkdir $stagingArea unless -d $stagingArea;

   move( "$localProjectDirectory/$project", "$stagingArea/$project" );

   my $md5File = "$stagingArea/$project" . $md5suffix;

   open DATA,">$md5File" or die "Could not create md5sum file [$md5File]: $!\n";

   print DATA "$md5\n";

   close DATA;

   return;

}

# verifies the project is correct on the server by comparing the checksums

# calculated locally and remote server. If valid, moves it into the final

# location on the remote server

sub validateTarget {

   my ( $remoteServer, $remoteStaging, $remoteTarget, $project, $checksum ) = @_;

   my $md5sum = `ssh $remoteServer "find '$remoteStaging/$project' -type f -exec md5sum \\{\\} \\; | cut -d' ' -f1 | sort | md5sum | cut -d' ' -f1"`;

   chomp $md5sum;

   if ( $checksum eq $md5sum ) {

      my $command = "ssh $remoteServer \"mv '$remoteStaging/$project' '$remoteTarget'\"";

      if ( system( $command ) == 0 ) {

         return 1;

      } else {

         &logit( "Unable to move $project to $remoteServer:$remoteTarget" );

         return 0;

      }

   } else {

      &logit( "Invalid checksum moving project $project" );

      return 0;

   }

}

# reads the checksum file

sub getCheckSum {

   my ( $project, $staging )  = @_;

   $project .= $md5suffix;

   if ( open DATA, "<$staging/$project" ) {

      my $cksum = <DATA>;

      chomp $cksum;

      close DATA;

      return $cksum;

   } 

   &logit( "Could not open $staging/$project: $!" );

   return '';

}

# simple little logger that records some information   

sub logit {

   my $message = shift;

   my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);

   my $now = sprintf( "%04d-%02d-%02d %02d:%-2d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec );

   open LOG, ">>/tmp/archiveProjects.log" or die "could not write to archiveProjects.log: $!\n";

   print LOG "$now\t$message\n";   

   close LOG;

}

# simply remove everything from the trash directory

sub cleanTrash {

   my $trashDir = shift;

   `rm -fR $trashDir/*`;

}

#&cleanTrash( $trashDirectory ) if &getProjectsToMove( $trashDirectory );

# first, check and see if we have any projects we need to move

@ProjectsToMove = &getProjects( $localProjectDirectory );

foreach my $project ( @ProjectsToMove ) {

   my $md5 = &calcMD5( "$localProjectDirectory/$project" );

   &logit( "New Project $md5\t$project" );

   &moveToStaging( $project, $stagingArea, $md5 );

}

# done with that, now we need to see if there is anything in the staging area

# that needs to be sent to the remote server

opendir( my $dh, $stagingArea ) or die "Could not read $stagingArea: $!\n";

my @directories;

my @toMove = grep { /$md5suffix$/ } readdir( $dh );

foreach my $project ( @toMove ) {

   $project =~ m/^(.*)\.md5sum/;

   $project = $1;

   my $md5sum = &getCheckSum( $project, $stagingArea );

   next unless $md5sum;

   my $rsync = "rsync -av '$stagingArea/$project' $targetServer:$targetStagingArea/ > /tmp/lastrsync.log";

   &logit( $rsync );

   if ( system ( $rsync ) == 0 ) { # we succeeded

      if ( &validateTarget( $targetServer, $targetStagingArea, $targetProjectDirectory, $project, $md5sum ) ) {

         `mkdir -p $trashDirectory` unless -d $trashDirectory;

         move( "$stagingArea/$project", "$trashDirectory/$project" );

         $project .= $md5suffix;

         move( "$stagingArea/$project", "$trashDirectory/$project" );

         &logit( "Successfully moved project $project to $targetServer" );

      } else {

         &logit( "Unable to validate target for $project" );

      }

   } else {

      &logit( "Unknown error attempting to rsync $project" );

   }

}

1;