WebSVN – zfs_utils – Diff – /trunk/replication/replicate

 #! /usr/bin/env perl
+# replicate
+# Author: R. W. Rodolico
 # very simple script to replicate a ZFS snapshot to another server.
 # no fancy bells and whistles, does not create snapshots, and does
 # not prune them. No major error checking either
+#
+# This is free software, and may be redistributed under the same terms
+#
+# Copyright (c) 2025, R. W. Rodolico
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.[13]
+#
+# version 1.0.0 20250614 RWR
+# Added support for inconsistent child dataset snapshots
+# If all child datasets do not have all the snapshots the parent has,
+# then we break the job into multiple jobs, one for each dataset
 use strict;
 use warnings;
 use Data::Dumper;
 use Getopt::Long;
 Getopt::Long::Configure ("bundling");
+# define the version number
+# see https://metacpan.org/pod/release/JPEACOCK/version-0.97/lib/version.pod
+use version 0.77; our $VERSION = version->declare("v1.0.0");
 # create our configuration, with some defaults
 # these are overridden by command line stuff
 my $config = {
    # the source, where we're coming from
    'source' => '',
    'recurse' => 0,
    # show more information
    'verbose' => 0
    };
+# Parses a dataset string, which may include a server (server:dataset),
+# and returns a hashref with 'server' and 'dataset' keys.
 sub parseDataSet {
    my $data = shift;
    my %return;
    my ( $server, $dataset ) = split( ':', $data );
    if ( $dataset ) { # they passed a server:dataset
       $return{'dataset'} = $server;
+   }
    return \%return;
+}
+# Appends log messages to /tmp/replicate.log.
 sub logit {
    open LOG, ">>/tmp/replicate.log" or die "Could not open replicate.log: $!\n";
    print LOG join( "\n", @_ ) .  "\n";
    close LOG;
+}
-# runs a command, redirecting stderr to stdout (which it ignores)
+# Runs a shell command, capturing stdout and stderr.
-# then returns 0 and $output on success.
-# if error, returns error code and string describing error
+# Returns (0, output) on success, or (error_code, error_message) on failure.
 sub run {
    my $command = shift;
    #&logit( $command );
    my $output = qx/$command 2>&1/;
    if ($? == -1) {
       return ($? >> 8, sprintf "child exited with value %d", $? >> 8 ) if $? >> 8;
+   }
    return (0,$output);
+}
+# Retrieves all ZFS snapshots for a given dataset (and server, if remote).
+# Filters snapshots by a regex pattern, and returns a hashref of snapshots
+# with metadata (key, refer, used).
 sub getSnaps {
    my ($config,$pattern) = @_;
    my %return;
    # actual command to run to get all snapshots, recursively, of the dataset
    my $command = 'zfs list -r -t snap ' . $config->{'dataset'};
       $return{$dataset}{'snaps'}{$snap}{'used'} = $used;
+   }
    return \%return;
+}
-# get tne number of bytes we will be syncing.
+# Calculates the number of bytes that would be transferred for the next sync.
+# Returns the size in bytes, or 0 if datasets are up to date.
 sub findSize {
    my $config = shift;
    # check for new snapshots to sync. If they are equal, we are up to date
    if ( $config->{'source'}->{'lastSnap'} ne $config->{'target'}->{'lastSnap'} ) {
       # Build the source command
    } else { # nothing to sync
       return 0;
+   }
+}
-# create the command necessary to do the replication
+# Builds the shell command(s) needed to replicate the ZFS snapshot(s)
+# from source to target, using zfs send/receive and optionally pv.
 sub createCommands {
    my $config = shift;
    # check for new snapshots to sync. If they are equal, we are up to date
    if ( $config->{'source'}->{'lastSnap'} ne $config->{'target'}->{'lastSnap'} ) {
       # Build the source command
    } else { # source and target are in sync, so do nothing
       return '# Nothing new to sync';
+   }
+}
-# find the last snapshot in a hash. The hash is assumed to have a subkey
+# Finds the most recent snapshot in a hash of snapshots.
-# 'key'. look for the largest subkey, and return the key for it
+# Returns the snapshot name with the largest 'key' value.
 sub getLastSnapshot {
    my $snapList = shift;
    my $lastKey = 0;
    my $lastSnap = '';
    foreach my $snap ( keys %$snapList ) {
+      }
+   }
    return $lastSnap;
+}
+# Checks if all child datasets have all the snapshots the parent has.
+# If not, returns a list of datasets to replicate individually.
+sub check_child_snap_consistency {
+    my ($config, $side) = @_;
+    my $snaps = $config->{$side}{'snapshots'};
+    my @datasets = keys %$snaps;
+    return @datasets if @datasets == 1; # Only parent, nothing to check
+    my $parent = (sort @datasets)[0]; # Assume parent is first (no / in name or shortest)
+    my %parent_snaps = %{ $snaps->{$parent}{'snaps'} };
+    my @inconsistent;
+    foreach my $child (@datasets) {
+        next if $child eq $parent;
+        foreach my $snap (keys %parent_snaps) {
+            unless (exists $snaps->{$child}{'snaps'}{$snap}) {
+                push @inconsistent, $child;
+                last;
+            }
+        }
+    }
+    if (@inconsistent) {
+        # Return all datasets as separate jobs
+        return @datasets;
+    } else {
+        # All children have all parent snaps, treat as one job
+        return ($parent);
+    }
+}
+# Calculates the last snapshot for source and target, and checks for consistency.
+# Returns (source_last_snap, target_last_snap, warnings_arrayref).
 sub calculate {
    my $config = shift;
    my @warnings;
    } else {
       return( '','',\@warnings);
+   }
 } # sub calculate
+# Prints usage/help message and exits.
 sub help {
    use File::Basename;
    my $me = fileparse( $0 );
    my $helpMessage = <<"   EOF";
       $me [flags] [source [target]]
+      Version $VERSION
          Syncs source dataset to target dataset
       Parameters (optional)
          source - dataset syncing from
          target - dataset syncing to
          --filter|f  - Filter (regex) to limit source snapshots to process
          --dryrun|n  - Only displays command(s) to be run
          --recurse|r - Process dataset and all child datasets
          --verbose|v - increase verbosity of output
          --bwlimit   - Limit the speed of the connect to # bytes/s. KMGT allowed
+         --version|V - display the version number and exit
       May use short flags with bundling, ie -nrvv is valid for
       --dryrun --recurse --verbose --verbose
       Either source or target must contain a DNS name or IP address of a remote
    'filter|f=s',
    'dryrun|n',
    'recurse|r',
    'bwlimit=s',
    'verbose|v+',
+   'version|V',
    'help|h'
 );
 &help() if $config->{'help'};
+if ($config->{'version'}) {
+   print "replicate version $VERSION\n" ;
+   exit 0;
+}
 # allow them to use positional, without flags, such as
 # replicate source target --filter='regex' -n
 $config->{'source'} = shift unless $config->{'source'};
 $config->{'target'} = shift unless $config->{'target'};
 die "You must enter a source and a target, at a minimum\n" unless $config->{'source'} && $config->{'target'};
 # connect to servers and get all existing snapshots
 $config->{'target'}->{'snapshots'} = &getSnaps( $config->{'target'}, $config->{'filter'} );
 $config->{'source'}->{'snapshots'} = &getSnaps( $config->{'source'}, $config->{'filter'} );
-# we sync from last snap on target machine to last snap on source machine. calculate simply
+# Check for child dataset snapshot consistency on source and target
-# finds the last snapshot on source and target
+my @source_jobs = check_child_snap_consistency($config, 'source');
-( $config->{'source'}->{'lastSnap'}, $config->{'target'}->{'lastSnap'} ) = &calculate( $config );
+my @target_jobs = check_child_snap_consistency($config, 'target');
-# calculate transfer size if they want any feedback at all. Since this does take a few seconds
+# If either side has inconsistencies, break into per-dataset jobs
+my %all_jobs;
+$all_jobs{$_}++ for (@source_jobs, @target_jobs);
+foreach my $dataset (sort keys %all_jobs) {
-# to calculate, we won't run it unless they want a report
+    # Prepare a config for this dataset
+    my %job_config = %$config;
+    $job_config{'source'} = { %{$config->{'source'}}, 'dataset' => $config->{'source'}{'dataset'} . $dataset, 'snapshots' => { $dataset => $config->{'source'}{'snapshots'}{$dataset} } };
-$config->{'report'}->{'Bytes Transferred'} = &findSize( $config ) if $config->{'verbose'};
+    $job_config{'target'} = { %{$config->{'target'}}, 'dataset' => $config->{'target'}{'dataset'} . $dataset, 'snapshots' => { $dataset => $config->{'target'}{'snapshots'}{$dataset} } };
+    ( $job_config{'source'}{'lastSnap'}, $job_config{'target'}{'lastSnap'} ) = &calculate( \%job_config );
-# actually creates the commands to do the replicate
+    $job_config{'report'}{'Bytes Transferred'} = &findSize( \%job_config ) if $config->{'verbose'};
-my $commands = &createCommands( $config );
+    my $commands = &createCommands( \%job_config );
-print "$commands\n" if $config->{'verbose'} > 1 or $config->{'dryrun'};
+    print "$commands\n" if $config->{'verbose'} > 1 or $config->{'dryrun'};
-if ( $config->{'dryrun'} ) {
+    if ( $config->{'dryrun'} ) {
-   print "Dry Run\n";
+        print "Dry Run for $dataset\n";
-} else {
+    } else {
-   print qx/$commands/ if $commands =~ m/^[a-zA-Z]/;
+        print qx/$commands/ if $commands =~ m/^[a-zA-Z]/;
+    }
+}
 $config->{'report'}->{'End Time'} = time;
 $config->{'report'}->{'Elapsed Time'} = $config->{'report'}->{'End Time'} - $config->{'report'}->{'Start Time'};
 if ( $config->{'verbose'}  ) {

Subversion Repositories zfs_utils

(root)/trunk/replication/replicate – Rev 18 → 20