WebSVN – zfs_utils – Diff – /trunk/replication/replicate

    # the source, where we're coming from
    'source' => '',
    # the target, where we want to replicate to
    'target' => '',
    # compile the regex
-   'filter' => qr/(\d{4}.\d{2}.\d{2}.\d{2}.\d{2})/,
+   'filter' => '(\d{4}.\d{2}.\d{2}.\d{2}.\d{2})',
    # if non-zero, just display the commands we'd use, don't run them
    'dryrun' => 0,
    # whether to do all child datasets also (default)
-   'recurse' => 1,
+   'recurse' => 0,
    # show more information
    'verbose' => 0
    };
 sub parseDataSet {
       $return{$dataset}{'snaps'}{$snap}{'used'} = $used;
+   }
    return \%return;
+}
-#sub diffSnaps {
-#   my ( $config->{'source'}, $config->{'target'} ) = @_;
-#   my @source = sort keys %$config->{'source'};
+# get tne number of bytes we will be syncing.
-#   my @target = sort keys %$config->{'target'};
-#   print "===Source\n" . join( "\n", @source ) . "\n===Target\n" . join( "\n", @target ) . "\n";
-#   my $s = 0;
+sub findSize {
-#   my $t = 0;
+   my $config = shift;
-#   my %return;
-#   $return{'deleteTarget'} = [];
+   # check for new snapshots to sync. If they are equal, we are up to date
-#   $return{'addTarget'} = [];
+   if ( $config->{'source'}->{'lastSnap'} ne $config->{'target'}->{'lastSnap'} ) {
-#   $return{'lastMatch'} = 0;
+      # Build the source command
-#   $return{'finalSync'} = 0;
+      my $sourceCommand = sprintf( '%s@%s %s@%s',
-#   while ( $s < @source && $t < @target ) {
+                               $config->{'source'}->{'dataset'},
-#      if ( $config->{'source'}[$s] eq $config->{'target'}[$t] ) { # matchies, just keep going
+                               $config->{'target'}->{'lastSnap'},
-#         print "Source $s [$config->{'source'}[$s]] matches target $t [$config->{'target'}[$t]]\n";
+                               $config->{'source'}->{'dataset'},
-#         $return{'lastMatch'} = $config->{'source'}[$s]; # keep track of the largest match
+                               $config->{'source'}->{'lastSnap'}
-#         $s++; $t++;
+                           );
-#      } elsif ( $config->{'target'}[$t] ne $config->{'source'}[$s] ) { # we are processing stuff that needs to be deleted on target
+      # prepend 'zfs send' and the flags. Note that verbose is only for the one which is local
+      $sourceCommand = 'zfs send -' .
-#         push @{$return{'deleteTarget'}}, $config->{'target'}[$t];
+                  ( $config->{'recurse'} ? 'R' : '' ) . # recurse if they asked for it
-#         print "Adding delete target $t [$config->{'target'}[$t]]\n";
+                  # turn on verbose if they asked for level 2 AND if source is local
-#         $t++;
+                  'Pwn' .
-#      }
-#   }
-#   die "Could not reconcile snapshots, ran out of source too soon\n" if $s > @source;
+                  # this is the part that asks for incremental
-#   # put a value into finalSync to make sure there is one. If we do not have any sync
+                  'I ' .
-#   # to do, final and lastMatch will be the same
+                  $sourceCommand;
-#   $return{'finalSync'} = $return{'lastMatch'};
+      # wrap the ssh call if this is remote
-#   while ( $s < @source ) {
+      $sourceCommand = "ssh $config->{source}->{server} '$sourceCommand'" if  $config->{'source'}->{'server'};
-#      push @{$return{'addTarget'}}, $config->{'source'}[$s];
+      print "Checking Size with\n$sourceCommand\n" if $config->{'verbose'} > 2;
-#      $return{'finalSync'} = $config->{'source'}[$s];
+      my ( $error, $output ) = &run( $sourceCommand );
-#      $s++;
+      return -1 if $error;
-#   }
+      # the size is the second column (tab separated) of the last line (\n separated) in $output
-#   die Dumper( \%return );
+      return (
-#   return \%return;
+               split(
-#}
-#sub arrayEquals {
+                  "\t",
-#   my ($a, $b ) = @_;
+                  (
-#   return 0 unless @{$a} == @{$b}; # they are different sizes
+                     split( "\n", $output )
-#   for ( my $i = 0; $i < @$a; $i++ ) {
+                  )[-1]
-#      if ( $$a[$i] ne $$b[$i] ) {
+               )
-#         print STDERR "No Match!\n" . join( "\t", @$a ) . "\n" . join( "\t", @$b ) . "\n";
-#         return 0;
+            )[1];
+   } else { # nothing to sync
-#      }
+      return 0;
-#   }
+   }
-#   return 1;
-#}
+}
+# create the command necessary to do the replication
 sub createCommands {
-   my ( $lastSource, $lastTarget, $config ) = @_;
-   my @return;
+   my $config = shift;
-   # check for new snapshots to sync
+   # check for new snapshots to sync. If they are equal, we are up to date
+   if ( $config->{'source'}->{'lastSnap'} ne $config->{'target'}->{'lastSnap'} ) {
-   if ( $lastSource ne $lastTarget ) {
+      # Build the source command
-      my $snapRange = $config->{'target'}->{'dataset'} . '@' . $config->{'target'};
+      my $sourceCommand = sprintf( '%s@%s %s@%s',
-      my $flags = ( $config->{'recurse'} ? 'R' : '' ) .
+                               $config->{'source'}->{'dataset'},
-                  ( $config->{'verbose'} == 2 ? 'v' : '' );
+                               $config->{'target'}->{'lastSnap'},
-      my $source = $config->{'source'}->{'dataset'} . '@' . $lastSource;
+                               $config->{'source'}->{'dataset'},
-      my $target = $config->{'target'}->{'dataset'};
+                               $config->{'source'}->{'lastSnap'}
-      # first create the replicate command. The send command request recursion (-R)
+                           );
-      # and the range of snapshots including all intermediate ones (-I)
+      # prepend 'zfs send' and the flags. Note that verbose is only for the one which is local
-      my $command = 'zfs send -RI ';
+      $sourceCommand = 'zfs send -' .
-      $command .= $config->{'target'}->{'dataset'} . '@' . $config->{'target'} . ' ';
+                  ( $config->{'recurse'} ? 'R' : '' ) . # recurse if they asked for it
+                  # turn on verbose if they asked for level 2 AND if source is local
-      $command .= $config->{'source'}->{'dataset'} . '@' . $config->{'source'};
+                  ( $config->{'verbose'} > 1 && ! $config->{'source'}->{'server'} ? 'v' : '' ) .
+                  # this is the part that asks for incremental
+                  'I ' .
+                  $sourceCommand;
+      # wrap the ssh call if this is remote
-      $command = "ssh $config->{source}->{server} '$config->{'source'}Command'" if $config->{'source'}->{'server'};
+      $sourceCommand = "ssh $config->{source}->{server} '$sourceCommand'" if  $config->{'source'}->{'server'};
+      # Now, build the target command
-      $command = 'zfs receive -v ';
+      my $targetCommand = 'zfs receive ' .
+                          ( ! $config->{'target'}->{'server'} && $config->{'verbose'} > 1 ? '-v ' : '') .
-      $command .= $config->{'target'}->{'dataset'};
+                          $config->{'target'}->{'dataset'};
-      $command = "ssh $config->{target}->{server} '$config->{'source'}Command'" if $config->{'target'}->{'server'};
+      $targetCommand = "ssh $config->{target}->{server} '$targetCommand'" if  $config->{'target'}->{'server'};
+      # return the command
-      push @return, $command . ' | ' . $command;
+      return $sourceCommand . ' | ' . $targetCommand;
-   } else {
+   } else { # source and target are in sync, so do nothing
-      push @return, '# Nothing new to sync';
+      return '# Nothing new to sync';
+   }
-   # now, check for snapshots to remove
-   #if ( $config->{'actions'}->{'deleteTarget'} ) {
-   #   my $delete = $config->{'actions'}->{'deleteTarget'};
-   #   foreach my $ds ( @$delete ) {
-   #      push @return, "zfs destroy -r $config->{target}->{'dataset'}\@$ds";
-   #   }
-   #} else {
-   #   push @return, "# No old snapshots to be removed";
-   #}
-   return \@return;
+}
 # find the last snapshot in a hash. The hash is assumed to have a subkey
 # 'key'. look for the largest subkey, and return the key for it
 sub getLastSnapshot {
    } else {
       return( '','',\@warnings);
+   }
 } # sub calculate
+sub help {
+   use File::Basename;
+   my $me = fileparse( $0 );
+   my $helpMessage = <<"   EOF";
+      $me [flags] [source [target]]
+         Syncs source dataset to target dataset
+      Parameters (optional)
+         source - dataset syncing from
+         target - dataset syncing to
+      Flags
+         --source|s  - Alternate way to pass source dataset
+         --target|t  - Alternate way to pass target dataset
+         --filter|f  - Filter (regex) to limit source snapshots to process
+         --dryrun|n  - Only displays command(s) to be run
+         --recurse|r - Process dataset and all child datasets
+         --verbose|v - increase verbosity of output
+      May use short flags with bundling, ie -nrvv is valid for
+      --dryrun --recurse --verbose --verbose
+      Either source or target must contain a DNS name or IP address of a remote
+      machine, separated from the dataset with a colon, ie
+         --source fbsd:storage/mydata
+      would use the dataset storage/mydata on the server fbsd. The other dataset
+      is assumed to be the local machine
+      filter is a string which is a valid regular expression. Only snapshots matching
+      that string will be used from the source dataset
+      By default, only error messages are displayed. verbose will display statistics
+      on size and transfer time. Invoking twice will display entire output of
+      send/receive (whichever is the local machine)
+   EOF
+   # get rid of indentation
+   $helpMessage =~ s/^      //;
+   $helpMessage =~ s/\n      /\n/g;
+   print $helpMessage;
+   exit 1;
+} # help
 GetOptions( $config,
    'source|s=s',
    'target|t=s',
    'filter|f=s',
    'dryrun|n',
    'recurse|r',
-   'verbose|v',
+   'verbose|v+',
    'help|h'
 );
+&help() if $config->{'help'};
 # allow them to use positional, without flags, such as
 # replicate source target --filter='regex' -n
 $config->{'source'} = shift unless $config->{'source'};
 $config->{'target'} = shift unless $config->{'target'};
 die "You must enter a source and a target, at a minimum\n" unless $config->{'source'} && $config->{'target'};
+# keep track of when we started this run
+$config->{'report'}->{'Start Time'} = time;
 # WARNING: this converts source and targets from a string to a hash
 # '10.0.0.1:data/set' becomes ( 'server' => '10.0.0.1', 'dataset' => 'data/set')
 # and 'data/set' becomes ( 'server' => '', 'dataset' => 'data/set')
 $config->{'source'} = &parseDataSet( $config->{'source'} );
 $config->{'target'} = &parseDataSet( $config->{'target'} );
 # both source and target can not have a server portion; one must be local
 die "Source and Target can not both be remote\n" if $config->{'source'}->{'server'} && $config->{'target'}->{'server'};
-$config->{'source'}->{'snapshots'} = &getSnaps( $config->{'source'}, $config->{'filter'} );
+# connect to servers and get all existing snapshots
 $config->{'target'}->{'snapshots'} = &getSnaps( $config->{'target'}, $config->{'filter'} );
+$config->{'source'}->{'snapshots'} = &getSnaps( $config->{'source'}, $config->{'filter'} );
-# we sync from last snap on target machine to last snap on source machine
+# we sync from last snap on target machine to last snap on source machine. calculate simply
+# finds the last snapshot on source and target
-my ( $lastSource, $lastTarget ) = &calculate( $config );
+( $config->{'source'}->{'lastSnap'}, $config->{'target'}->{'lastSnap'} ) = &calculate( $config );
+# calculate transfer size if they want any feedback at all. Since this does take a few seconds
+# to calculate, we won't run it unless they want a report
-#print Dumper( $config ) . "\nSource = $lastSource\nTarget = $lastTarget\n"; die;
+$config->{'report'}->{'Bytes Transferred'} = &findSize( $config ) if $config->{'verbose'};
 # actually creates the commands to do the replicate
-my $commands = &createCommands( $lastSource, $lastTarget, $config );
+my $commands = &createCommands( $config );
+print "$commands\n" if $config->{'verbose'} or $config->{'dryrun'};
-for ( my $i = 0; $i < @{$commands}; $i++ ) {
+if ( $config->{'dryrun'} ) {
+   print "Dry Run\n";
+} else {
-   print "$$commands[$i]\n" if $config->{'verbose'} or $config->{'dryrun'};
+   print qx/$commands/ if $commands =~ m/^[a-zA-Z]/;
+}
+$config->{'report'}->{'End Time'} = time;
+$config->{'report'}->{'Elapsed Time'} = $config->{'report'}->{'End Time'} - $config->{'report'}->{'Start Time'};
+if ( $config->{'verbose'} ) {
    if ( $config->{'dryrun'} ) {
-      print "Dry Run\n";
+      print "Would have transferred $config->{'report'}->{'Bytes Transferred'} bytes\n";
    } else {
-      print qx/$$commands[$i]/ if $$commands[$i] =~ m/^[a-zA-Z]/;
+      print "Transferred $config->{'report'}->{'Bytes Transferred'} bytes in $config->{'report'}->{'Elapsed Time'} seconds\n";
+   }
+}
-#print Dumper( $config );
 ;

Subversion Repositories zfs_utils

(root)/trunk/replication/replicate – Rev 5 → 6