| Line 8... |
Line 8... |
| 8 |
my $source = shift;
|
8 |
my $source = shift;
|
| 9 |
my $target = shift;
|
9 |
my $target = shift;
|
| 10 |
|
10 |
|
| 11 |
die "Usage: replicate source target\n" unless $source && $target;
|
11 |
die "Usage: replicate source target\n" unless $source && $target;
|
| 12 |
|
12 |
|
| 13 |
my $dryRun = 0; # if set, will only display the command to be executed
|
13 |
my $dryRun = 1; # if set, will only display the command to be executed
|
| 14 |
|
14 |
|
| 15 |
my $config = {
|
15 |
my $config = {
|
| 16 |
# compile the regex
|
16 |
# compile the regex
|
| 17 |
'pattern' => qr/auto-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}/
|
17 |
'pattern' => qr/(\d{4}.\d{2}.\d{2}.\d{2}.\d{2})/
|
| 18 |
};
|
18 |
};
|
| 19 |
|
19 |
|
| 20 |
sub parseDataSet {
|
20 |
sub parseDataSet {
|
| 21 |
my $data = shift;
|
21 |
my $data = shift;
|
| 22 |
my %return;
|
22 |
my %return;
|
| Line 53... |
Line 53... |
| 53 |
my ($config,$pattern) = @_;
|
53 |
my ($config,$pattern) = @_;
|
| 54 |
my %return;
|
54 |
my %return;
|
| 55 |
# actual command to run to get all snapshots, recursively, of the dataset
|
55 |
# actual command to run to get all snapshots, recursively, of the dataset
|
| 56 |
my $command = 'zfs list -r -t snap ' . $config->{'dataset'};
|
56 |
my $command = 'zfs list -r -t snap ' . $config->{'dataset'};
|
| 57 |
$command = "ssh $config->{server} '$command'" if $config->{'server'};
|
57 |
$command = "ssh $config->{server} '$command'" if $config->{'server'};
|
| - |
|
58 |
#die "$command\n";
|
| 58 |
my ($error, $output ) = &run( $command );
|
59 |
my ($error, $output ) = &run( $command );
|
| 59 |
die $output if $error;
|
60 |
#die "Error running $command with output\n$output" if $error;
|
| 60 |
my @snaps = split( "\n", $output );
|
61 |
my @snaps = split( "\n", $output );
|
| 61 |
chomp @snaps;
|
62 |
chomp @snaps;
|
| 62 |
for (my $i = 0; $i < @snaps; $i++ ) {
|
63 |
for (my $i = 0; $i < @snaps; $i++ ) {
|
| 63 |
# parse out the space delmited fields
|
64 |
# parse out the space delmited fields
|
| 64 |
my ($fullname, $used, $avail, $refer, $mount) = split( /\s+/, $snaps[$i] );
|
65 |
my ($fullname, $used, $avail, $refer, $mount) = split( /\s+/, $snaps[$i] );
|
| Line 66... |
Line 67... |
| 66 |
my ($dataset, $snap) = split( '@', $fullname );
|
67 |
my ($dataset, $snap) = split( '@', $fullname );
|
| 67 |
# remove the root dataset name
|
68 |
# remove the root dataset name
|
| 68 |
$dataset =~ s/^$config->{'dataset'}//;
|
69 |
$dataset =~ s/^$config->{'dataset'}//;
|
| 69 |
# skip anything not matching our regex
|
70 |
# skip anything not matching our regex
|
| 70 |
next unless $pattern && $snap && $snap =~ m/$pattern/;
|
71 |
next unless $pattern && $snap && $snap =~ m/$pattern/;
|
| - |
|
72 |
# grab the matched key
|
| - |
|
73 |
$return{$dataset}{'snaps'}{$snap}{'key'} = $1;
|
| - |
|
74 |
# and remove all non-numerics
|
| - |
|
75 |
$return{$dataset}{'snaps'}{$snap}{'key'} =~ s/[^0-9]//g;
|
| - |
|
76 |
# get the transfer size
|
| 71 |
$return{$dataset}{'snaps'}{$snap}{'refer'} = $refer;
|
77 |
$return{$dataset}{'snaps'}{$snap}{'refer'} = $refer;
|
| - |
|
78 |
# get the actual disk space used
|
| 72 |
$return{$dataset}{'snaps'}{$snap}{'used'} = $used;
|
79 |
$return{$dataset}{'snaps'}{$snap}{'used'} = $used;
|
| 73 |
}
|
80 |
}
|
| 74 |
return \%return;
|
81 |
return \%return;
|
| 75 |
}
|
82 |
}
|
| 76 |
|
83 |
|
| Line 122... |
Line 129... |
| 122 |
}
|
129 |
}
|
| 123 |
return 1;
|
130 |
return 1;
|
| 124 |
}
|
131 |
}
|
| 125 |
|
132 |
|
| 126 |
sub createCommands {
|
133 |
sub createCommands {
|
| 127 |
my $config = shift;
|
134 |
my ( $source, $target, $config ) = @_;
|
| 128 |
my @return;
|
135 |
my @return;
|
| 129 |
# check for new snapshots to sync
|
136 |
# check for new snapshots to sync
|
| 130 |
if ( $config->{'actions'}->{'lastMatch'} ne $config->{'actions'}->{'finalSync'} ) {
|
137 |
if ( $source ne $target ) {
|
| 131 |
# first create the replicate command. The send command request recursion (-R)
|
138 |
# first create the replicate command. The send command request recursion (-R)
|
| 132 |
# and the range of snapshots including all intermediate ones (-I)
|
139 |
# and the range of snapshots including all intermediate ones (-I)
|
| 133 |
my $sourceCommand = 'zfs send -RI ';
|
140 |
my $sourceCommand = 'zfs send -RI ';
|
| 134 |
$sourceCommand .= $config->{'source'}->{'dataset'} . '@' . $config->{'actions'}->{'lastMatch'} . ' ';
|
141 |
$sourceCommand .= $config->{'source'}->{'dataset'} . '@' . $target . ' ';
|
| 135 |
$sourceCommand .= $config->{'source'}->{'dataset'} . '@' . $config->{'actions'}->{'finalSync'};
|
142 |
$sourceCommand .= $config->{'source'}->{'dataset'} . '@' . $source;
|
| 136 |
$sourceCommand = "ssh $config->{source}->{server} '$sourceCommand'" if $config->{'source'}->{'server'};
|
143 |
$sourceCommand = "ssh $config->{source}->{server} '$sourceCommand'" if $config->{'source'}->{'server'};
|
| 137 |
|
144 |
|
| 138 |
my $targetCommand = 'zfs receive -v ';
|
145 |
my $targetCommand = 'zfs receive -v ';
|
| 139 |
$targetCommand .= $config->{'target'}->{'dataset'};
|
146 |
$targetCommand .= $config->{'target'}->{'dataset'};
|
| 140 |
$targetCommand = "ssh $config->{target}->{server} '$sourceCommand'" if $config->{'target'}->{'server'};
|
147 |
$targetCommand = "ssh $config->{target}->{server} '$sourceCommand'" if $config->{'target'}->{'server'};
|
| 141 |
push @return, $sourceCommand . ' | ' . $targetCommand;
|
148 |
push @return, $sourceCommand . ' | ' . $targetCommand;
|
| 142 |
} else {
|
149 |
} else {
|
| 143 |
push @return, '# Nothing new to sync';
|
150 |
push @return, '# Nothing new to sync';
|
| 144 |
}
|
151 |
}
|
| 145 |
# now, check for snapshots to remove
|
152 |
# now, check for snapshots to remove
|
| 146 |
if ( $config->{'actions'}->{'deleteTarget'} ) {
|
153 |
#if ( $config->{'actions'}->{'deleteTarget'} ) {
|
| 147 |
my $delete = $config->{'actions'}->{'deleteTarget'};
|
154 |
# my $delete = $config->{'actions'}->{'deleteTarget'};
|
| 148 |
foreach my $ds ( @$delete ) {
|
155 |
# foreach my $ds ( @$delete ) {
|
| 149 |
push @return, "zfs destroy -r $config->{target}->{'dataset'}\@$ds";
|
156 |
# push @return, "zfs destroy -r $config->{target}->{'dataset'}\@$ds";
|
| 150 |
}
|
157 |
# }
|
| 151 |
} else {
|
158 |
#} else {
|
| 152 |
push @return, "# No old snapshots to be removed";
|
159 |
# push @return, "# No old snapshots to be removed";
|
| 153 |
}
|
160 |
#}
|
| 154 |
return \@return;
|
161 |
return \@return;
|
| 155 |
}
|
162 |
}
|
| 156 |
|
163 |
|
| - |
|
164 |
# find the last snapshot in a hash. The hash is assumed to have a subkey
|
| - |
|
165 |
# 'key'. look for the largest subkey, and return the key for it
|
| - |
|
166 |
sub getLastSnapshot {
|
| - |
|
167 |
my $snapList = shift;
|
| - |
|
168 |
my $lastKey = 0;
|
| - |
|
169 |
my $lastSnap = '';
|
| - |
|
170 |
foreach my $snap ( keys %$snapList ) {
|
| - |
|
171 |
if ( $snapList->{$snap}->{'key'} > $lastKey ) {
|
| - |
|
172 |
$lastKey = $snapList->{$snap}->{'key'};
|
| - |
|
173 |
$lastSnap = $snap;
|
| - |
|
174 |
}
|
| - |
|
175 |
}
|
| - |
|
176 |
return $lastSnap;
|
| - |
|
177 |
}
|
| 157 |
|
178 |
|
| 158 |
|
179 |
|
| 159 |
sub calculate {
|
180 |
sub calculate {
|
| 160 |
my $config = shift;
|
181 |
my $config = shift;
|
| - |
|
182 |
|
| 161 |
my $return;
|
183 |
my @warnings;
|
| 162 |
my $allMatch;
|
184 |
|
| - |
|
185 |
# find the last snapshot date in each dataset, on each target
|
| 163 |
my $lastMatch;
|
186 |
foreach my $machine ( 'source', 'target' ) {
|
| - |
|
187 |
$config->{$machine}->{'last'} = 0; # track the last entry in all children in dataset
|
| - |
|
188 |
$config->{$machine}->{'allOk'} = 1; # assumed to be true, becomes false if some children do not have snapshots
|
| 164 |
foreach my $dataset ( sort keys %{$config->{'source'}->{'snapshots'}} ) {
|
189 |
foreach my $child ( keys %{ $config->{$machine}->{'snapshots'} } ) {
|
| 165 |
next unless exists $config->{'source'}->{'snapshots'}->{$dataset};
|
190 |
$config->{$machine}->{'snapshots'}->{$child}->{'last'} =
|
| 166 |
die "No matching target for $dataset\n" unless $config->{'target'}->{'snapshots'}->{$dataset};
|
191 |
&getLastSnapshot( $config->{$machine}->{'snapshots'}->{$child}->{'snaps'} );
|
| 167 |
$return->{$dataset} = &diffSnaps(
|
192 |
# set the machine last if we haven't done so yet
|
| - |
|
193 |
$config->{$machine}->{'last'} = $config->{$machine}->{'snapshots'}->{$child}->{'last'} unless $config->{$machine}->{'last'};
|
| 168 |
$config->{'source'}->{'snapshots'}->{$dataset}->{'snaps'},
|
194 |
# keep track of the last snapshot for each set
|
| 169 |
$config->{'target'}->{'snapshots'}->{$dataset}->{'snaps'}
|
195 |
if ( $config->{$machine}->{'last'} ne $config->{$machine}->{'snapshots'}->{$child}->{'last'} ) {
|
| 170 |
);
|
- |
|
| 171 |
$allMatch = $return->{$dataset} unless $allMatch;
|
196 |
$config->{$machine}->{'allOk'} = 0;
|
| 172 |
# die Dumper( $allMatch ) . "\n";
|
197 |
push @warnings, "Warning: $machine does not have consistent snapshots at $child";;
|
| 173 |
next;
|
198 |
}
|
| 174 |
unless (
|
199 |
}
|
| - |
|
200 |
}
|
| - |
|
201 |
# make sure the source has a corresponding snap for target->last
|
| 175 |
&arrayEquals( $return->{'allMatch'}->{'deleteTarget'}, $return->{$dataset}->{'deleteTarget'} ) &&
|
202 |
foreach my $child ( keys %{ $config->{'target'}->{'snapshots'} } ) {
|
| 176 |
&arrayEquals( $return->{'allMatch'}->{'addTarget'}, $return->{$dataset}->{'addTarget'} )
|
203 |
if (! exists ($config->{'source'}->{'snapshots'}->{$child}->{'snaps'}->{$config->{'target'}->{'snapshots'}->{$child}->{'last'}} ) ) {
|
| 177 |
) {
|
204 |
$config->{'source'}->{'allOk'} = 0;
|
| 178 |
warn "Warning: dataset $dataset does not match\n";
|
205 |
push @warnings, "Warning: We do not have consistent snapshots";
|
| 179 |
last;
|
- |
|
| 180 |
}
|
206 |
}
|
| 181 |
}
|
207 |
}
|
| 182 |
#print Dumper( $allMatch );
|
208 |
my $return;
|
| 183 |
$return->{'lastMatch'} = $allMatch->{'lastMatch'};
|
- |
|
| 184 |
$return->{'finalSync'} = $allMatch->{'finalSync'};
|
- |
|
| 185 |
# die Dumper( $allMatch->{'deleteTarget'} ) . "\n" ;
|
209 |
if ( $config->{'source'}->{'allOk'} and $config->{'target'}->{'allOk'} ) { # whew, they match
|
| 186 |
$return->{'deleteTarget'} = $allMatch->{'deleteTarget'};
|
210 |
return( $config->{'source'}->{'last'}, $config->{'target'}->{'last'}, \@warnings );
|
| 187 |
# print Dumper( $return ) . "\n"; die;
|
- |
|
| 188 |
return $return;
|
211 |
} else {
|
| 189 |
#print Dumper( $return ) . "\n"; die;
|
212 |
return( '','',\@warnings);
|
| - |
|
213 |
}
|
| 190 |
}
|
214 |
}
|
| 191 |
|
215 |
|
| 192 |
$config->{'source'} = &parseDataSet( $source );
|
216 |
$config->{'source'} = &parseDataSet( $source );
|
| 193 |
$config->{'target'} = &parseDataSet( $target );
|
217 |
$config->{'target'} = &parseDataSet( $target );
|
| 194 |
|
218 |
|
| Line 196... |
Line 220... |
| 196 |
die "Source and Target can not both be remote\n" if $config->{'source'}->{'server'} && $config->{'target'}->{'server'};
|
220 |
die "Source and Target can not both be remote\n" if $config->{'source'}->{'server'} && $config->{'target'}->{'server'};
|
| 197 |
|
221 |
|
| 198 |
$config->{'source'}->{'snapshots'} = &getSnaps( $config->{'source'}, $config->{'pattern'} );
|
222 |
$config->{'source'}->{'snapshots'} = &getSnaps( $config->{'source'}, $config->{'pattern'} );
|
| 199 |
$config->{'target'}->{'snapshots'} = &getSnaps( $config->{'target'}, $config->{'pattern'} );
|
223 |
$config->{'target'}->{'snapshots'} = &getSnaps( $config->{'target'}, $config->{'pattern'} );
|
| 200 |
|
224 |
|
| 201 |
$config->{'actions'} = &calculate( $config );
|
225 |
# $config->{'actions'} = &calculate( $config );
|
| - |
|
226 |
my ( $lastSource, $lastTarget ) = &calculate( $config );
|
| 202 |
|
227 |
|
| 203 |
#print Dumper( $config ); die;
|
228 |
#print Dumper( $config ) . "\nSource = $lastSource\nTarget = $lastTarget\n"; die;
|
| 204 |
|
229 |
|
| 205 |
my $commands = &createCommands( $config );
|
230 |
my $commands = &createCommands( $lastSource, $lastTarget, $config );
|
| 206 |
for ( my $i = 0; $i < @{$commands}; $i++ ) {
|
231 |
for ( my $i = 0; $i < @{$commands}; $i++ ) {
|
| 207 |
print "$$commands[$i]\n";
|
232 |
print "$$commands[$i]\n";
|
| 208 |
if ( $dryRun ) {
|
233 |
if ( $dryRun ) {
|
| 209 |
print "Dry Run\n";
|
234 |
print "Dry Run\n";
|
| 210 |
} else {
|
235 |
} else {
|
| Line 212... |
Line 237... |
| 212 |
}
|
237 |
}
|
| 213 |
}
|
238 |
}
|
| 214 |
|
239 |
|
| 215 |
#print Dumper( $config );
|
240 |
#print Dumper( $config );
|
| 216 |
|
241 |
|
| 217 |
1;
|
- |
|
| 218 |
|
242 |
1;
|
| - |
|
243 |
|