96 |
rodolico |
1 |
#! /usr/bin/env perl
|
|
|
2 |
|
98 |
rodolico |
3 |
# snapShot: Manage ZFS snapshots
|
|
|
4 |
# see http://wiki.linuxservertech.com for additional information
|
|
|
5 |
# Copyright (C) 2022 R. W. Rodolico
|
|
|
6 |
#
|
107 |
rodolico |
7 |
# version 1.0, 20220423 RWR
|
98 |
rodolico |
8 |
# Initial Release
|
|
|
9 |
#
|
107 |
rodolico |
10 |
# version 1.0.1 20220430 RWR
|
|
|
11 |
# Removed some debugging, set so it will always log the actions to /tmp/snapShot
|
98 |
rodolico |
12 |
#
|
119 |
rodolico |
13 |
# version 1.0.2 20220529 RWR
|
|
|
14 |
# Fixed error where 86400 was mistyped 864000, replacing all instance of numbers other than 60 and 3600 with contants
|
|
|
15 |
# Added DEBUG flag in config which will turn on TESTING, the dump some additional information to screen during run
|
|
|
16 |
#
|
98 |
rodolico |
17 |
# This program is free software: you can redistribute it and/or modify
|
|
|
18 |
# it under the terms of the GNU General Public License as published by
|
|
|
19 |
# the Free Software Foundation, either version 3 of the License, or
|
|
|
20 |
# (at your option) any later version.
|
|
|
21 |
#
|
|
|
22 |
# This program is distributed in the hope that it will be useful,
|
|
|
23 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
24 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
25 |
# GNU General Public License for more details.
|
|
|
26 |
#
|
|
|
27 |
# You should have received a copy of the GNU General Public License
|
|
|
28 |
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
29 |
#
|
103 |
rodolico |
30 |
# Warning, this script requires non-standard Perl modules YAML::Tiny and Hash::Merge
|
98 |
rodolico |
31 |
# Under Debian: apt install libyaml-tiny-perl libhash-merge-simple-perl
|
|
|
32 |
# Under FreeBSD: cpan -i Hash::Merge::Simple YAML::Tiny
|
|
|
33 |
|
|
|
34 |
|
96 |
rodolico |
35 |
use strict;
|
|
|
36 |
use warnings;
|
|
|
37 |
|
119 |
rodolico |
38 |
use version; our $VERSION = version->declare( 'v1.0.2');
|
96 |
rodolico |
39 |
use Data::Dumper;
|
|
|
40 |
use Time::Local;
|
|
|
41 |
use POSIX qw(strftime);
|
|
|
42 |
use YAML::Tiny; # apt-get libyaml-tiny-perl under debian, BSD Systems: cpan -i YAML::Tiny
|
98 |
rodolico |
43 |
use Hash::Merge::Simple qw/ merge clone_merge /; # apt install libhash-merge-simple-perl or cpan -i Hash::Merge::Simple
|
96 |
rodolico |
44 |
|
|
|
45 |
|
|
|
46 |
# globals
|
|
|
47 |
my $CONFIG_FILE_NAME = 'snapShot.yaml';
|
119 |
rodolico |
48 |
my $SECONDS_PER_DAY = 86400;
|
|
|
49 |
my $SECONDS_PER_WEEK = 7 * $SECONDS_PER_DAY;
|
|
|
50 |
my $SECONDS_PER_MONTH = 30.5 * $SECONDS_PER_DAY;
|
|
|
51 |
my $SECONDS_PER_YEAR = 365.2425 * $SECONDS_PER_DAY;
|
96 |
rodolico |
52 |
|
|
|
53 |
# This will be read in from snapShot.yaml
|
|
|
54 |
my $config;
|
|
|
55 |
|
|
|
56 |
#
|
|
|
57 |
# find where the script is actually located as cfg should be there
|
|
|
58 |
#
|
|
|
59 |
sub getScriptLocation {
|
|
|
60 |
use strict;
|
|
|
61 |
use File::Spec::Functions qw(rel2abs);
|
|
|
62 |
use File::Basename;
|
|
|
63 |
return dirname(rel2abs($0));
|
|
|
64 |
}
|
|
|
65 |
|
|
|
66 |
#
|
|
|
67 |
# Read the configuration file from current location
|
|
|
68 |
# and return it as a string
|
|
|
69 |
#
|
|
|
70 |
sub readConfig {
|
|
|
71 |
my $scriptLocation = &getScriptLocation();
|
|
|
72 |
if ( -e "$scriptLocation/$CONFIG_FILE_NAME" ) {
|
|
|
73 |
my $yaml = YAML::Tiny->read( "$scriptLocation/$CONFIG_FILE_NAME" );
|
|
|
74 |
# use clone_merge to merge conf file into $config
|
|
|
75 |
# overwrites anything in $config if it exists in the config file
|
|
|
76 |
$config = clone_merge( $config, $yaml->[0] );
|
|
|
77 |
return 1;
|
|
|
78 |
}
|
|
|
79 |
return 0;
|
|
|
80 |
}
|
|
|
81 |
|
|
|
82 |
|
|
|
83 |
# parse one single line from the output of `zfs list [-t snapshot]`
|
|
|
84 |
sub parseListing {
|
|
|
85 |
my ($line,$keys) = @_;
|
|
|
86 |
chomp $line;
|
|
|
87 |
my %values;
|
|
|
88 |
@values{@$keys} = split( /\s+/, $line );
|
|
|
89 |
return \%values;
|
|
|
90 |
}
|
|
|
91 |
|
|
|
92 |
|
|
|
93 |
# this will parse the date out of the snapshots and put the values into
|
|
|
94 |
# the hash {'date'}
|
|
|
95 |
sub parseSnapshots {
|
|
|
96 |
my ( $snapShots, $config) = @_;
|
|
|
97 |
my $keys = $config->{'snapshot'}->{'parseFields'};
|
|
|
98 |
foreach my $snapShot ( keys %$snapShots ) {
|
|
|
99 |
my %temp;
|
|
|
100 |
# run the regex, capture the output to an array, then populate the hash %temp
|
|
|
101 |
# using the regex results as the values, and $keys as the keys
|
|
|
102 |
@temp{@$keys} = ( $snapShot =~ m/$config->{'snapshot'}->{'parse'}/ );
|
|
|
103 |
# while we're here, calculate the unix time (epoch). NOTE: month is 0 based
|
|
|
104 |
$temp{'unix'} = timelocal( 0,$temp{'minute'},$temp{'hour'},$temp{'day'},$temp{'month'}-1,$temp{'year'} );
|
|
|
105 |
# put this into our record
|
|
|
106 |
$snapShots->{$snapShot}->{'date'} = \%temp;
|
|
|
107 |
}
|
|
|
108 |
}
|
|
|
109 |
|
|
|
110 |
# run $command, then parse its output and return the results as a hashref
|
103 |
rodolico |
111 |
# $command is one of zfs list or zfs list -t snapshot
|
|
|
112 |
# In other words, get all datasets/volumes or get all snapshots
|
96 |
rodolico |
113 |
sub getListing {
|
|
|
114 |
my ($configuration, $regex, $command ) = @_;
|
|
|
115 |
my %dataSets;
|
|
|
116 |
|
103 |
rodolico |
117 |
# get all datasets/volumes or snapshots
|
96 |
rodolico |
118 |
my @zfsList = `$command`;
|
|
|
119 |
foreach my $thisSet ( @zfsList ) {
|
103 |
rodolico |
120 |
# parse the line into its portions. The only one we use right now is name
|
96 |
rodolico |
121 |
my $temp = &parseListing( $thisSet, $configuration->{'listingKeys'} );
|
103 |
rodolico |
122 |
if ( $temp->{'name'} =~ m/^($regex)$/ ) { # it matches the regex we're using, so save it
|
96 |
rodolico |
123 |
$dataSets{$temp->{'name'}} = $temp;
|
|
|
124 |
}
|
|
|
125 |
}
|
103 |
rodolico |
126 |
return \%dataSets; # return all entries we are looking for
|
96 |
rodolico |
127 |
}
|
|
|
128 |
|
119 |
rodolico |
129 |
# will convert something like 1 day to the number of seconds ($SECONDS_PER_DAY) for math.
|
96 |
rodolico |
130 |
# month and year are approximations (30.5 day = a month, 365.2425 days is a year)
|
101 |
rodolico |
131 |
# For month and year, use the int function to convert back to integer
|
96 |
rodolico |
132 |
sub period2seconds {
|
|
|
133 |
my ($count, $unit) = ( shift =~ m/\s*(\d+)\s*([a-z]+)\s*/i );
|
|
|
134 |
$unit = lc $unit;
|
97 |
rodolico |
135 |
if ( $unit eq 'hour' ) {
|
96 |
rodolico |
136 |
$count *= 3600;
|
|
|
137 |
} elsif ( $unit eq 'day' ) {
|
119 |
rodolico |
138 |
$count *= $SECONDS_PER_DAY;
|
96 |
rodolico |
139 |
} elsif ( $unit eq 'week' ) {
|
119 |
rodolico |
140 |
$count *= $SECONDS_PER_WEEK;
|
96 |
rodolico |
141 |
} elsif ( $unit eq 'month' ) {
|
119 |
rodolico |
142 |
$count *= int( $SECONDS_PER_MONTH );
|
96 |
rodolico |
143 |
} elsif ( $unit eq 'year' ) {
|
119 |
rodolico |
144 |
$count *= int( $SECONDS_PER_YEAR );
|
96 |
rodolico |
145 |
} else {
|
|
|
146 |
die "Unknown units [$unit] in period2seconds\n";
|
|
|
147 |
}
|
|
|
148 |
return $count;
|
|
|
149 |
}
|
|
|
150 |
|
119 |
rodolico |
151 |
# quick convert unix time stamp to YYYY-MM-DD HH:MM
|
|
|
152 |
sub unix2Gregorian {
|
|
|
153 |
use POSIX qw(strftime);
|
|
|
154 |
my $timestamp = shift;
|
|
|
155 |
|
120 |
rodolico |
156 |
return strftime "%F %R", localtime $timestamp ;
|
119 |
rodolico |
157 |
}
|
|
|
158 |
|
120 |
rodolico |
159 |
# just converts a number of seconds to DD day HH:MM
|
119 |
rodolico |
160 |
|
|
|
161 |
sub secondsToHuman {
|
|
|
162 |
my $seconds = shift;
|
|
|
163 |
|
|
|
164 |
my $days = int( $seconds / $SECONDS_PER_DAY );
|
|
|
165 |
$seconds -= $days * $SECONDS_PER_DAY;
|
|
|
166 |
my $hours = int( $seconds / 60);
|
|
|
167 |
$seconds -= $hours * 60;
|
|
|
168 |
my $minutes = int( $seconds / 60 );
|
|
|
169 |
$seconds -= $minutes * 60;
|
120 |
rodolico |
170 |
return sprintf( '%02dD %02dH%02dM', $days, $hours, $minutes );
|
119 |
rodolico |
171 |
}
|
|
|
172 |
|
96 |
rodolico |
173 |
# Merges datasets, snapshots and some stuff from the configuration into the datasets
|
103 |
rodolico |
174 |
# hash. After this, $config and $snapshots should no longer be necessary
|
97 |
rodolico |
175 |
sub mergeData {
|
96 |
rodolico |
176 |
my ($datasets,$snapshots,$config) = @_;
|
|
|
177 |
my $confKeys = $config->{'datasets'};
|
|
|
178 |
foreach my $thisDataset ( keys %$datasets ) {
|
103 |
rodolico |
179 |
# go through each configuration entry and see if we match the current dataset
|
96 |
rodolico |
180 |
foreach my $conf (keys %$confKeys ) {
|
103 |
rodolico |
181 |
if ( $thisDataset =~ m/^$conf$/ ) { # found it, so store the configuration values into the dataset
|
96 |
rodolico |
182 |
$datasets->{$thisDataset}->{'recursive'} = $confKeys->{$conf}->{'recursive'};
|
|
|
183 |
$datasets->{$thisDataset}->{'frequency'} = &period2seconds( $confKeys->{$conf}->{'frequency'} );
|
|
|
184 |
$datasets->{$thisDataset}->{'retention'} = &period2seconds( $confKeys->{$conf}->{'retention'} );
|
103 |
rodolico |
185 |
last; # there is only one, so no need to process any more for this configuration key
|
96 |
rodolico |
186 |
} # if
|
|
|
187 |
} # foreach
|
103 |
rodolico |
188 |
# do the same for the snapshots we found; bind them to the data set
|
96 |
rodolico |
189 |
foreach my $snapshot ( keys %$snapshots ) {
|
|
|
190 |
if ( $snapshot =~ m/^$thisDataset@/ ) { # this is a match
|
|
|
191 |
# copy the snapshot into the dataset
|
|
|
192 |
$datasets->{$thisDataset}->{'snapshots'}->{$snapshot} = $snapshots->{$snapshot};
|
103 |
rodolico |
193 |
# track the latest snapshot (we use this to decide whether it is time to add a new one)
|
96 |
rodolico |
194 |
$datasets->{$thisDataset}->{'lastSnap'} = $snapshots->{$snapshot}->{'date'}->{'unix'}
|
|
|
195 |
if ! defined( $datasets->{$thisDataset}->{'lastSnap'} ) || $datasets->{$thisDataset}->{'lastSnap'} < $snapshots->{$snapshot}->{'date'}->{'unix'};
|
103 |
rodolico |
196 |
# delete the snapshot, to free up memory
|
96 |
rodolico |
197 |
delete $snapshots->{$snapshot};
|
|
|
198 |
} # if
|
|
|
199 |
} # foreach
|
|
|
200 |
} # foreach
|
97 |
rodolico |
201 |
} # sub mergeData
|
96 |
rodolico |
202 |
|
103 |
rodolico |
203 |
|
|
|
204 |
# check to see if a particular snapshot is ready to be destroyed, ie right now is greater than the retention period
|
|
|
205 |
# if $recurive is true, add the '-r' to the command to do a recursive destroy
|
96 |
rodolico |
206 |
sub checkRetention {
|
|
|
207 |
my ( $retentionPeriod, $recursive, $snapshots, $now ) = @_;
|
120 |
rodolico |
208 |
print "\tRemoving snapshots older than " . &unix2Gregorian( $now - $retentionPeriod ) . "\n" if $config->{'DEBUG'};
|
103 |
rodolico |
209 |
my @toDelete; # an array of destroy commands
|
96 |
rodolico |
210 |
foreach my $thisSnapshot ( keys %$snapshots ) {
|
|
|
211 |
# print "checking $thisSnapshot\n\tNow: $now\n\tDate: $snapshots->{$thisSnapshot}->{date}->{unix}\n\tRetention: $retentionPeriod\n\n";
|
103 |
rodolico |
212 |
if ( $now - $snapshots->{$thisSnapshot}->{'date'}->{'unix'} > $retentionPeriod ) { # it is too old
|
|
|
213 |
push ( @toDelete, ( 'zfs destroy ' . ($recursive ? '-r ' : '') . $thisSnapshot ) ); # list it to be destroyed
|
96 |
rodolico |
214 |
}
|
|
|
215 |
}
|
103 |
rodolico |
216 |
return @toDelete; # just return the list of destroy commands to be executed
|
96 |
rodolico |
217 |
}
|
|
|
218 |
|
103 |
rodolico |
219 |
|
|
|
220 |
# just return the command to create a new snapshot. Very simple, but I wanted the code to be isolated in case something needed
|
|
|
221 |
# to change. Basically, zfs snapshot [-r] datasetname@template
|
96 |
rodolico |
222 |
sub makeSnapshot {
|
|
|
223 |
my ( $datasetName, $recursive, $snapshotName ) = @_;
|
|
|
224 |
return
|
|
|
225 |
'zfs snapshot ' .
|
|
|
226 |
($recursive ? '-r ' : '') .
|
|
|
227 |
$datasetName . $snapshotName;
|
|
|
228 |
}
|
|
|
229 |
|
103 |
rodolico |
230 |
# this is the biggie; everything leads to here. We will take every dataset/volume we found, and decide whether some old snapshots
|
|
|
231 |
# need to be destroyed, and whether a new snapshot needs to be created.
|
96 |
rodolico |
232 |
sub process {
|
|
|
233 |
my ( $datasets, $now, $snapshotName, $slop ) = @_;
|
103 |
rodolico |
234 |
my @toDelete; # will hold all the destroy commands
|
|
|
235 |
my @toAdd; # will hold all the create commands
|
96 |
rodolico |
236 |
|
103 |
rodolico |
237 |
foreach my $thisDataset ( keys %$datasets ) { # Look at each dataset/volume in turn
|
119 |
rodolico |
238 |
if ( $config->{'DEBUG'} ) {
|
|
|
239 |
print "Found Datasset $thisDataset\n";
|
|
|
240 |
print "\tNext Snapshot due " .
|
|
|
241 |
&unix2Gregorian( $datasets->{$thisDataset}->{'lastSnap'} + $datasets->{$thisDataset}->{'frequency'} - $slop ) . "\n";
|
120 |
rodolico |
242 |
print "\tRetention period set to " . &secondsToHuman( $datasets->{$thisDataset}->{'retention'} ) . "\n";
|
119 |
rodolico |
243 |
}
|
103 |
rodolico |
244 |
# if any snapshots need to be destroyed, add them to @toDelete
|
96 |
rodolico |
245 |
push( @toDelete,
|
|
|
246 |
&checkRetention(
|
|
|
247 |
$datasets->{$thisDataset}->{'retention'},
|
|
|
248 |
$datasets->{$thisDataset}->{'recursive'},
|
|
|
249 |
$datasets->{$thisDataset}->{'snapshots'},
|
|
|
250 |
$now )
|
|
|
251 |
);
|
120 |
rodolico |
252 |
# if there is no snapshaor or it is time to add a new snapshot, add it to @toAdd
|
|
|
253 |
if ( !$datasets->{$thisDataset}->{'lastSnap'} || $datasets->{$thisDataset}->{'lastSnap'} + $datasets->{$thisDataset}->{'frequency'} - $slop < $now ) {
|
96 |
rodolico |
254 |
push @toAdd, &makeSnapshot( $thisDataset, $datasets->{$thisDataset}->{'recursive'}, $snapshotName )
|
|
|
255 |
}
|
|
|
256 |
}
|
103 |
rodolico |
257 |
# return the actions, deletions first, adds second (executed in that order)
|
96 |
rodolico |
258 |
return ( @toDelete, @toAdd );
|
|
|
259 |
}
|
|
|
260 |
|
103 |
rodolico |
261 |
# Run 0 or more commands
|
|
|
262 |
# the first thing on the stack is a flag for testing
|
|
|
263 |
# everything after that is an ordered list of commands to be executed.
|
|
|
264 |
# If any command fails, all subsequent commands abort
|
96 |
rodolico |
265 |
sub run {
|
|
|
266 |
my $testing = shift;
|
103 |
rodolico |
267 |
return 0 unless @_; # bail if there are no commands to run
|
107 |
rodolico |
268 |
# dump the run to /tmp so the user can see the last one written
|
|
|
269 |
open LOG, ">/tmp/snapShot" or die "could not write to /tmp/snapShot: $!\n";
|
|
|
270 |
print LOG join( "\n", @_ ) . "\n";
|
|
|
271 |
close LOG;
|
|
|
272 |
unless ( $testing ) { # run the commands if we're not testing
|
103 |
rodolico |
273 |
my $out; # capture all output
|
|
|
274 |
while ( my $command = shift ) { # for each command on the stack
|
|
|
275 |
$out .= `$command` . "\n"; # add it to $out
|
|
|
276 |
if ( $? ) { # we had an error, add debugging text, the end program
|
96 |
rodolico |
277 |
$out .= "Error executing command\n\t$command\n\t";
|
|
|
278 |
if ($? == -1) {
|
|
|
279 |
$out .= "failed to execute $command: $!";
|
|
|
280 |
} elsif ($? & 127) {
|
|
|
281 |
$out .= sprintf( "child died with signal %d, %s coredump", ($? & 127), ($? & 128) ? 'with' : 'without' );
|
|
|
282 |
} else {
|
|
|
283 |
$out .= sprintf( "child exited with value %d", $? >> 8 );
|
|
|
284 |
}
|
|
|
285 |
$out .= "\n";
|
|
|
286 |
return $out;
|
|
|
287 |
}
|
|
|
288 |
}
|
|
|
289 |
}
|
103 |
rodolico |
290 |
return 0; # we succeeded
|
96 |
rodolico |
291 |
}
|
|
|
292 |
|
99 |
rodolico |
293 |
&readConfig() or die "Could not read config file: $!\n";
|
119 |
rodolico |
294 |
# if debug set, be sure testing is set also, so no actions are taken
|
|
|
295 |
$config->{'TESTING'} = 1 if $config->{'DEBUG'};
|
97 |
rodolico |
296 |
|
103 |
rodolico |
297 |
# we're pre-calculating some things so we don't do it over and over for each entry
|
96 |
rodolico |
298 |
# grab the time once
|
|
|
299 |
my $now = time;
|
|
|
300 |
# create the string to be used for all snapshots, using $now and the template provided
|
|
|
301 |
my $snapshotName = '@' . strftime($config->{'snapshot'}->{'template'},localtime $now);
|
103 |
rodolico |
302 |
# Create the dataset regex by joing all of the regexes defined.
|
96 |
rodolico |
303 |
$config->{'dataset_regex'} = '(' . join( ')|(', keys %{ $config->{'datasets'} } ) . ')' unless $config->{'dataset_regex'};
|
|
|
304 |
#print $config{'dataset_regex'} . "\n";
|
|
|
305 |
$config->{'snapshot_regex'} = '(' . $config->{'dataset_regex'} . ')@' . $config->{'snapshot'}->{'parse'};
|
|
|
306 |
#print $config->{'snapshot_regex'} . "\n\n";
|
|
|
307 |
|
|
|
308 |
#die Dumper( $config ) . "\n";
|
|
|
309 |
# first, find all datasets which match our keys
|
|
|
310 |
my $dataSets = &getListing( $config, $config->{'dataset_regex'}, 'zfs list' );
|
|
|
311 |
# and, find all snapshots that match
|
|
|
312 |
my $snapshots = &getListing( $config, $config->{'snapshot_regex'}, 'zfs list -t snapshot' );
|
|
|
313 |
# get the date/time of the snapshots and store them in the hash
|
|
|
314 |
&parseSnapshots($snapshots, $config );
|
97 |
rodolico |
315 |
# mergeData the snapshots into the datasets for convenience
|
|
|
316 |
&mergeData( $dataSets, $snapshots, $config );
|
96 |
rodolico |
317 |
# Now, let's do the actual processing
|
|
|
318 |
my @commands = &process( $dataSets, $now, $snapshotName, &period2seconds( $config->{'slop'} ) );
|
99 |
rodolico |
319 |
#print join ( "\n", @commands ) . "\n";
|
96 |
rodolico |
320 |
my $errors;
|
99 |
rodolico |
321 |
print "Error: $errors\n" if $errors = &run( $config->{'TESTING'}, @commands );
|
96 |
rodolico |
322 |
|
99 |
rodolico |
323 |
# print Dumper( $dataSets );
|
96 |
rodolico |
324 |
#print Dumper( $snapshots );
|
|
|
325 |
|
|
|
326 |
#print join ("\n", sort keys( %$dataSets ) ) . "\n\n";
|
|
|
327 |
#print join( "\n", sort keys( %$snapshots ) ) . "\n";
|
|
|
328 |
|
|
|
329 |
1;
|