Subversion Repositories sysadmin_scripts

#! /usr/bin/env perl

use strict;

use warnings;

use Data::Dumper;

my $source = shift;

my $target = shift;

die "Usage: replicate source target\n" unless $source && $target;

my $dryRun = 1; # if set, will only display the command to be executed

my $config = {

   # compile the regex

   'pattern' => qr/(\d{4}.\d{2}.\d{2}.\d{2}.\d{2})/

   };

sub parseDataSet {

   my $data = shift;

   my %return;

   my ( $server, $dataset ) = split( ':', $data );

   if ( $dataset ) { # they passed a server:dataset

      $return{'server'} = $server;

      $return{'dataset'} = $dataset;

   } else { # only passing in dataset, so assume localhost

      $return{'server'} = '';

      $return{'dataset'} = $server;

   }

   return \%return;

}

# runs a command, redirecting stderr to stdout (which it ignores)

# then returns 0 and $output on success.

# if error, returns error code and string describing error

sub run {

   my $command = shift;

   my $output = qx/$command 2>&1/;

   if ($? == -1) {

      return (-1,"failed to execute: $!");

   } elsif ($? & 127) {

      return ($?, sprintf "child died with signal %d, %s coredump",

        ($? & 127),  ($? & 128) ? 'with' : 'without' );

   } else {

      return ($? >> 8, sprintf "child exited with value %d", $? >> 8 ) if $? >> 8;

   }

   return (0,$output);

}

sub getSnaps {

   my ($config,$pattern) = @_;

   my %return;

   # actual command to run to get all snapshots, recursively, of the dataset

   my $command = 'zfs list -r -t snap ' . $config->{'dataset'};

   $command = "ssh $config->{server} '$command'" if $config->{'server'};

   #die "$command\n";

   my ($error, $output ) = &run( $command );

   #die "Error running $command with output\n$output" if $error;

   my @snaps = split( "\n", $output );

   chomp @snaps;

   for (my $i = 0; $i < @snaps; $i++ ) {

      # parse out the space delmited fields

      my ($fullname, $used, $avail, $refer, $mount) = split( /\s+/, $snaps[$i] );

      # break the name into dataset and snapname

      my ($dataset, $snap) = split( '@', $fullname );

      # remove the root dataset name

      $dataset =~ s/^$config->{'dataset'}//;

      # skip anything not matching our regex

      next unless $pattern && $snap && $snap =~ m/$pattern/;

      # grab the matched key

      $return{$dataset}{'snaps'}{$snap}{'key'} = $1;

      # and remove all non-numerics

      $return{$dataset}{'snaps'}{$snap}{'key'} =~ s/[^0-9]//g;

      # get the transfer size

      $return{$dataset}{'snaps'}{$snap}{'refer'} = $refer;

      # get the actual disk space used

      $return{$dataset}{'snaps'}{$snap}{'used'} = $used;

   }

   return \%return;

}

sub diffSnaps {

   my ( $source, $target ) = @_;

   my @source = sort keys %$source;

   my @target = sort keys %$target;

#   print "===Source\n" . join( "\n", @source ) . "\n===Target\n" . join( "\n", @target ) . "\n";

   my $s = 0;

   my $t = 0;

   my %return;

   $return{'deleteTarget'} = [];

   $return{'addTarget'} = [];

   $return{'lastMatch'} = 0;

   $return{'finalSync'} = 0;

   while ( $s < @source && $t < @target ) {

      if ( $source[$s] eq $target[$t] ) { # matchies, just keep going

#         print "Source $s [$source[$s]] matches target $t [$target[$t]]\n";

         $return{'lastMatch'} = $source[$s]; # keep track of the largest match

         $s++; $t++;

      } elsif ( $target[$t] ne $source[$s] ) { # we are processing stuff that needs to be deleted on target

         push @{$return{'deleteTarget'}}, $target[$t];

#         print "Adding delete target $t [$target[$t]]\n";

         $t++;

      }

   }

   die "Could not reconcile snapshots, ran out of source too soon\n" if $s > @source;

   # put a value into finalSync to make sure there is one. If we do not have any sync

   # to do, final and lastMatch will be the same

   $return{'finalSync'} = $return{'lastMatch'};

   while ( $s < @source ) {

      push @{$return{'addTarget'}}, $source[$s];

      $return{'finalSync'} = $source[$s];

      $s++;

   }

#   die Dumper( \%return );

   return \%return;

}

sub arrayEquals {

   my ($a, $b ) = @_;

   return 0 unless @{$a} == @{$b}; # they are different sizes

   for ( my $i = 0; $i < @$a; $i++ ) {

      if ( $$a[$i] ne $$b[$i] ) {

         print STDERR "No Match!\n" . join( "\t", @$a ) . "\n" . join( "\t", @$b ) . "\n";

         return 0;

      }

   }

   return 1;

}

sub createCommands {

   my ( $source, $target, $config ) = @_;

   my @return;

   # check for new snapshots to sync

   if ( $source ne $target ) {

      # first create the replicate command. The send command request recursion (-R)

      # and the range of snapshots including all intermediate ones (-I)

      my $sourceCommand = 'zfs send -RI ';

      $sourceCommand .= $config->{'source'}->{'dataset'} . '@' . $target . ' ';

      $sourceCommand .= $config->{'source'}->{'dataset'} . '@' . $source;

      $sourceCommand = "ssh $config->{source}->{server} '$sourceCommand'" if $config->{'source'}->{'server'};

      my $targetCommand = 'zfs receive -v ';

      $targetCommand .= $config->{'target'}->{'dataset'};

      $targetCommand = "ssh $config->{target}->{server} '$sourceCommand'" if $config->{'target'}->{'server'};

      push @return, $sourceCommand . ' | ' . $targetCommand;

   } else {

      push @return, '# Nothing new to sync';

   }

   # now, check for snapshots to remove

   #if ( $config->{'actions'}->{'deleteTarget'} ) {

   #   my $delete = $config->{'actions'}->{'deleteTarget'};

   #   foreach my $ds ( @$delete ) {

   #      push @return, "zfs destroy -r $config->{target}->{'dataset'}\@$ds";

   #   }

   #} else {

   #   push @return, "# No old snapshots to be removed";

   #}

   return \@return;

}

# find the last snapshot in a hash. The hash is assumed to have a subkey

# 'key'. look for the largest subkey, and return the key for it

sub getLastSnapshot {

   my $snapList = shift;

   my $lastKey = 0;

   my $lastSnap = '';

   foreach my $snap ( keys %$snapList ) {

      if ( $snapList->{$snap}->{'key'} > $lastKey ) {

         $lastKey = $snapList->{$snap}->{'key'};

         $lastSnap = $snap;

      }

   }

   return $lastSnap;

}

sub calculate {

   my $config = shift;

   my @warnings;

   # find the last snapshot date in each dataset, on each target

   foreach my $machine ( 'source', 'target' ) {

      $config->{$machine}->{'last'} = 0; # track the last entry in all children in dataset

      $config->{$machine}->{'allOk'} = 1; # assumed to be true, becomes false if some children do not have snapshots

      foreach my $child ( keys %{ $config->{$machine}->{'snapshots'} } ) {

         $config->{$machine}->{'snapshots'}->{$child}->{'last'} = 

            &getLastSnapshot( $config->{$machine}->{'snapshots'}->{$child}->{'snaps'} );

         # set the machine last if we haven't done so yet

         $config->{$machine}->{'last'} = $config->{$machine}->{'snapshots'}->{$child}->{'last'} unless $config->{$machine}->{'last'};

         # keep track of the last snapshot for each set

         if ( $config->{$machine}->{'last'} ne $config->{$machine}->{'snapshots'}->{$child}->{'last'} ) {

            $config->{$machine}->{'allOk'} = 0;

            push @warnings, "Warning: $machine does not have consistent snapshots at $child";;

         }

      }

   }

   # make sure the source has a corresponding snap for target->last

   foreach my $child ( keys %{ $config->{'target'}->{'snapshots'} } ) {

      if (! exists ($config->{'source'}->{'snapshots'}->{$child}->{'snaps'}->{$config->{'target'}->{'snapshots'}->{$child}->{'last'}} ) ) {

         $config->{'source'}->{'allOk'} = 0;

         push @warnings, "Warning: We  do not have consistent snapshots";

      }

   }

   my $return;

   if ( $config->{'source'}->{'allOk'} and $config->{'target'}->{'allOk'} ) { # whew, they match

      return( $config->{'source'}->{'last'}, $config->{'target'}->{'last'}, \@warnings );

   } else {

      return( '','',\@warnings);

   }

}

$config->{'source'} = &parseDataSet( $source );

$config->{'target'} = &parseDataSet( $target );

# both source and target can not have a server portion; one must be local

die "Source and Target can not both be remote\n" if $config->{'source'}->{'server'} && $config->{'target'}->{'server'};

$config->{'source'}->{'snapshots'} = &getSnaps( $config->{'source'}, $config->{'pattern'} );

$config->{'target'}->{'snapshots'} = &getSnaps( $config->{'target'}, $config->{'pattern'} );

# $config->{'actions'} = &calculate( $config );

my ( $lastSource, $lastTarget ) = &calculate( $config );

#print Dumper( $config ) . "\nSource = $lastSource\nTarget = $lastTarget\n"; die;

my $commands = &createCommands( $lastSource, $lastTarget, $config );

for ( my $i = 0; $i < @{$commands}; $i++ ) {

   print "$$commands[$i]\n";

   if ( $dryRun ) {

      print "Dry Run\n";

   } else {

      print qx/$$commands[$i]/ if $$commands[$i] =~ m/^[a-zA-Z]/;

   }

}

#print Dumper( $config );

1;