Subversion Repositories sysadmin_scripts

Rev

Rev 119 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
96 rodolico 1
#! /usr/bin/env perl
2
 
98 rodolico 3
#    snapShot: Manage ZFS snapshots
4
#    see http://wiki.linuxservertech.com for additional information
5
#    Copyright (C) 2022  R. W. Rodolico
6
#
107 rodolico 7
#    version 1.0, 20220423 RWR
98 rodolico 8
#       Initial Release
9
#
107 rodolico 10
#    version 1.0.1 20220430 RWR
11
#       Removed some debugging, set so it will always log the actions to /tmp/snapShot
98 rodolico 12
#
119 rodolico 13
#    version 1.0.2 20220529 RWR
14
#       Fixed error where 86400 was mistyped 864000, replacing all instance of numbers other than 60 and 3600 with contants
15
#       Added DEBUG flag in config which will turn on TESTING, the dump some additional information to screen during run
16
#
98 rodolico 17
#    This program is free software: you can redistribute it and/or modify
18
#    it under the terms of the GNU General Public License as published by
19
#    the Free Software Foundation, either version 3 of the License, or
20
#    (at your option) any later version.
21
#
22
#    This program is distributed in the hope that it will be useful,
23
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
24
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
25
#    GNU General Public License for more details.
26
#
27
#    You should have received a copy of the GNU General Public License
28
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
29
#
103 rodolico 30
# Warning, this script requires non-standard Perl modules YAML::Tiny and Hash::Merge
98 rodolico 31
# Under Debian:  apt install libyaml-tiny-perl libhash-merge-simple-perl
32
# Under FreeBSD: cpan -i Hash::Merge::Simple YAML::Tiny
33
 
34
 
96 rodolico 35
use strict;
36
use warnings;
37
 
119 rodolico 38
use version; our $VERSION = version->declare( 'v1.0.2');
96 rodolico 39
use Data::Dumper;
40
use Time::Local;
41
use POSIX qw(strftime);
42
use YAML::Tiny; # apt-get libyaml-tiny-perl under debian, BSD Systems: cpan -i YAML::Tiny
98 rodolico 43
use Hash::Merge::Simple qw/ merge clone_merge /; # apt install libhash-merge-simple-perl or cpan -i Hash::Merge::Simple
96 rodolico 44
 
45
 
46
# globals
47
my $CONFIG_FILE_NAME = 'snapShot.yaml';
119 rodolico 48
my $SECONDS_PER_DAY = 86400;
49
my $SECONDS_PER_WEEK = 7 * $SECONDS_PER_DAY;
50
my $SECONDS_PER_MONTH = 30.5 * $SECONDS_PER_DAY;
51
my $SECONDS_PER_YEAR = 365.2425 * $SECONDS_PER_DAY;
96 rodolico 52
 
53
# This will be read in from snapShot.yaml
54
my $config;
55
 
56
#
57
# find where the script is actually located as cfg should be there
58
#
59
sub getScriptLocation {
60
   use strict;
61
   use File::Spec::Functions qw(rel2abs);
62
   use File::Basename;
63
   return dirname(rel2abs($0));
64
}
65
 
66
#
67
# Read the configuration file from current location 
68
# and return it as a string
69
#
70
sub readConfig {
71
   my $scriptLocation = &getScriptLocation();
72
   if ( -e "$scriptLocation/$CONFIG_FILE_NAME" ) {
73
      my $yaml = YAML::Tiny->read( "$scriptLocation/$CONFIG_FILE_NAME" );
74
      # use clone_merge to merge conf file into $config
75
      # overwrites anything in $config if it exists in the config file
76
      $config = clone_merge( $config, $yaml->[0] );
77
      return 1;
78
   }
79
   return 0;
80
}
81
 
82
 
83
# parse one single line from the output of `zfs list [-t snapshot]`
84
sub parseListing {
85
   my ($line,$keys) = @_;
86
   chomp $line;
87
   my %values;
88
   @values{@$keys} = split( /\s+/, $line );
89
   return \%values;
90
}      
91
 
92
 
93
# this will parse the date out of the snapshots and put the values into
94
# the hash {'date'}
95
sub parseSnapshots {
96
   my ( $snapShots, $config) = @_;
97
   my $keys = $config->{'snapshot'}->{'parseFields'};
98
   foreach my $snapShot ( keys %$snapShots ) {
99
      my %temp;
100
      # run the regex, capture the output to an array, then populate the hash %temp
101
      # using the regex results as the values, and $keys as the keys
102
      @temp{@$keys} = ( $snapShot =~ m/$config->{'snapshot'}->{'parse'}/ );
103
      # while we're here, calculate the unix time (epoch). NOTE: month is 0 based
104
      $temp{'unix'} = timelocal( 0,$temp{'minute'},$temp{'hour'},$temp{'day'},$temp{'month'}-1,$temp{'year'} );
105
      # put this into our record
106
      $snapShots->{$snapShot}->{'date'} = \%temp;
107
   }
108
}
109
 
110
# run $command, then parse its output and return the results as a hashref
103 rodolico 111
# $command is one of zfs list or zfs list -t snapshot
112
# In other words, get all datasets/volumes or get all snapshots
96 rodolico 113
sub getListing {
114
   my ($configuration, $regex, $command )  = @_;
115
   my %dataSets;
116
 
103 rodolico 117
   # get all datasets/volumes or snapshots
96 rodolico 118
   my @zfsList = `$command`;
119
   foreach my $thisSet ( @zfsList ) {
103 rodolico 120
      # parse the line into its portions. The only one we use right now is name
96 rodolico 121
      my $temp = &parseListing( $thisSet, $configuration->{'listingKeys'} );
103 rodolico 122
      if (  $temp->{'name'} =~ m/^($regex)$/ ) { # it matches the regex we're using, so save it
96 rodolico 123
         $dataSets{$temp->{'name'}} = $temp;
124
      }
125
   }
103 rodolico 126
   return \%dataSets; # return all entries we are looking for
96 rodolico 127
}
128
 
119 rodolico 129
# will convert something like 1 day to the number of seconds ($SECONDS_PER_DAY) for math.
96 rodolico 130
# month and year are approximations (30.5 day = a month, 365.2425 days is a year)
101 rodolico 131
# For month and year, use the int function to convert back to integer
96 rodolico 132
sub period2seconds {
133
   my ($count, $unit) = ( shift =~ m/\s*(\d+)\s*([a-z]+)\s*/i );
134
   $unit = lc $unit;
97 rodolico 135
   if ( $unit eq 'hour' ) {
96 rodolico 136
      $count *= 3600;
137
   } elsif ( $unit eq 'day' ) {
119 rodolico 138
      $count *= $SECONDS_PER_DAY;
96 rodolico 139
   } elsif ( $unit eq 'week' ) {
119 rodolico 140
      $count *= $SECONDS_PER_WEEK;
96 rodolico 141
   } elsif ( $unit eq 'month' ) {
119 rodolico 142
      $count *= int( $SECONDS_PER_MONTH );
96 rodolico 143
   } elsif ( $unit eq 'year' ) {
119 rodolico 144
      $count *= int( $SECONDS_PER_YEAR );
96 rodolico 145
   } else {
146
      die "Unknown units [$unit] in period2seconds\n";
147
   }
148
   return $count;
149
}
150
 
119 rodolico 151
# quick convert unix time stamp to YYYY-MM-DD HH:MM
152
sub unix2Gregorian {
153
   use POSIX qw(strftime);
154
   my $timestamp = shift;
155
 
120 rodolico 156
   return strftime "%F %R", localtime $timestamp ;
119 rodolico 157
}
158
 
120 rodolico 159
# just converts a number of seconds to DD day HH:MM
119 rodolico 160
 
161
sub secondsToHuman {
162
   my $seconds = shift;
163
 
164
   my $days = int( $seconds / $SECONDS_PER_DAY );
165
   $seconds -= $days * $SECONDS_PER_DAY;
166
   my $hours = int( $seconds / 60);
167
   $seconds -= $hours * 60;
168
   my $minutes = int( $seconds / 60 );
169
   $seconds -= $minutes * 60;
120 rodolico 170
   return sprintf( '%02dD %02dH%02dM', $days, $hours, $minutes );
119 rodolico 171
}
172
 
96 rodolico 173
# Merges datasets, snapshots and some stuff from the configuration into the datasets
103 rodolico 174
# hash. After this, $config and $snapshots should no longer be necessary
97 rodolico 175
sub mergeData {
96 rodolico 176
   my ($datasets,$snapshots,$config) = @_;
177
   my $confKeys = $config->{'datasets'};
178
   foreach my $thisDataset ( keys %$datasets ) {
103 rodolico 179
      # go through each configuration entry and see if we match the current dataset
96 rodolico 180
      foreach my $conf (keys %$confKeys ) {
103 rodolico 181
         if ( $thisDataset =~ m/^$conf$/ ) { # found it, so store the configuration values into the dataset
96 rodolico 182
            $datasets->{$thisDataset}->{'recursive'} = $confKeys->{$conf}->{'recursive'};
183
            $datasets->{$thisDataset}->{'frequency'} = &period2seconds( $confKeys->{$conf}->{'frequency'} );
184
            $datasets->{$thisDataset}->{'retention'} = &period2seconds( $confKeys->{$conf}->{'retention'} );
103 rodolico 185
            last; # there is only one, so no need to process any more for this configuration key
96 rodolico 186
         } # if
187
      } # foreach
103 rodolico 188
      # do the same for the snapshots we found; bind them to the data set
96 rodolico 189
      foreach my $snapshot ( keys %$snapshots ) {
190
         if ( $snapshot =~ m/^$thisDataset@/ ) { # this is a match
191
            # copy the snapshot into the dataset
192
            $datasets->{$thisDataset}->{'snapshots'}->{$snapshot} = $snapshots->{$snapshot};
103 rodolico 193
            # track the latest snapshot (we use this to decide whether it is time to add a new one)
96 rodolico 194
            $datasets->{$thisDataset}->{'lastSnap'} = $snapshots->{$snapshot}->{'date'}->{'unix'}
195
               if ! defined( $datasets->{$thisDataset}->{'lastSnap'} ) || $datasets->{$thisDataset}->{'lastSnap'} < $snapshots->{$snapshot}->{'date'}->{'unix'};
103 rodolico 196
            # delete the snapshot, to free up memory
96 rodolico 197
            delete $snapshots->{$snapshot};
198
         } # if
199
      } # foreach
200
   } # foreach
97 rodolico 201
} # sub mergeData
96 rodolico 202
 
103 rodolico 203
 
204
# check to see if a particular snapshot is ready to be destroyed, ie right now is greater than the retention period
205
# if $recurive is true, add the '-r' to the command to do a recursive destroy
96 rodolico 206
sub checkRetention {
207
   my ( $retentionPeriod, $recursive, $snapshots, $now ) = @_;
120 rodolico 208
   print "\tRemoving snapshots older than " . &unix2Gregorian( $now - $retentionPeriod ) . "\n"  if $config->{'DEBUG'};
103 rodolico 209
   my @toDelete; # an array of destroy commands
96 rodolico 210
   foreach my $thisSnapshot ( keys %$snapshots ) {
211
      # print "checking $thisSnapshot\n\tNow: $now\n\tDate: $snapshots->{$thisSnapshot}->{date}->{unix}\n\tRetention: $retentionPeriod\n\n";
103 rodolico 212
      if ( $now - $snapshots->{$thisSnapshot}->{'date'}->{'unix'} > $retentionPeriod ) { # it is too old
213
         push ( @toDelete, ( 'zfs destroy ' . ($recursive ? '-r ' : '') . $thisSnapshot ) ); # list it to be destroyed
96 rodolico 214
      }
215
   }
103 rodolico 216
   return @toDelete; # just return the list of destroy commands to be executed
96 rodolico 217
}   
218
 
103 rodolico 219
 
220
# just return the command to create a new snapshot. Very simple, but I wanted the code to be isolated in case something needed
221
# to change. Basically, zfs snapshot [-r] datasetname@template
96 rodolico 222
sub makeSnapshot {
223
   my ( $datasetName, $recursive, $snapshotName ) = @_;
224
   return 
225
      'zfs snapshot ' . 
226
      ($recursive ? '-r ' : '') . 
227
      $datasetName . $snapshotName;
228
}
229
 
103 rodolico 230
# this is the biggie; everything leads to here. We will take every dataset/volume we found, and decide whether some old snapshots
231
# need to be destroyed, and whether a new snapshot needs to be created.
96 rodolico 232
sub process {
233
   my ( $datasets, $now, $snapshotName, $slop ) = @_;
103 rodolico 234
   my @toDelete; # will hold all the destroy commands
235
   my @toAdd; # will hold all the create commands
96 rodolico 236
 
103 rodolico 237
   foreach my $thisDataset ( keys %$datasets ) { # Look at each dataset/volume in turn
119 rodolico 238
      if ( $config->{'DEBUG'} ) {
239
         print "Found Datasset $thisDataset\n";
240
         print "\tNext Snapshot due " . 
241
            &unix2Gregorian( $datasets->{$thisDataset}->{'lastSnap'} + $datasets->{$thisDataset}->{'frequency'} - $slop ) . "\n";
120 rodolico 242
         print "\tRetention period set to " . &secondsToHuman( $datasets->{$thisDataset}->{'retention'} ) . "\n";
119 rodolico 243
      }
103 rodolico 244
      # if any snapshots need to be destroyed, add them to @toDelete
96 rodolico 245
      push( @toDelete, 
246
         &checkRetention( 
247
         $datasets->{$thisDataset}->{'retention'}, 
248
         $datasets->{$thisDataset}->{'recursive'}, 
249
         $datasets->{$thisDataset}->{'snapshots'}, 
250
         $now )
251
         );
120 rodolico 252
      # if there is no snapshaor or it is time to add a new snapshot, add it to @toAdd
253
      if ( !$datasets->{$thisDataset}->{'lastSnap'} || $datasets->{$thisDataset}->{'lastSnap'} + $datasets->{$thisDataset}->{'frequency'} - $slop < $now ) {
96 rodolico 254
         push @toAdd, &makeSnapshot( $thisDataset, $datasets->{$thisDataset}->{'recursive'}, $snapshotName )
255
      }
256
   }
103 rodolico 257
   # return the actions, deletions first, adds second (executed in that order)
96 rodolico 258
   return ( @toDelete, @toAdd );
259
}   
260
 
103 rodolico 261
# Run 0 or more commands
262
# the first thing on the stack is a flag for testing
263
# everything after that is an ordered list of commands to be executed.
264
# If any command fails, all subsequent commands abort
96 rodolico 265
sub run {
266
   my $testing = shift;
103 rodolico 267
   return 0 unless @_; # bail if there are no commands to run
107 rodolico 268
   # dump the run to /tmp so the user can see the last one written
269
   open LOG, ">/tmp/snapShot" or die "could not write to /tmp/snapShot: $!\n";
270
   print LOG join( "\n", @_ ) . "\n";
271
   close LOG;
272
   unless ( $testing ) { # run the commands if we're not testing
103 rodolico 273
      my $out; # capture all output
274
      while ( my $command = shift ) { # for each command on the stack
275
         $out .= `$command` . "\n"; # add it to $out
276
         if ( $? ) { # we had an error, add debugging text, the end program
96 rodolico 277
            $out .= "Error executing command\n\t$command\n\t";
278
            if ($? == -1) {
279
                $out .= "failed to execute $command: $!";
280
            } elsif ($? & 127) {
281
                $out .= sprintf( "child died with signal %d, %s coredump", ($? & 127),  ($? & 128) ? 'with' : 'without' );
282
            } else {
283
                $out .= sprintf( "child exited with value %d", $? >> 8 );
284
            }
285
            $out .= "\n";
286
            return $out;
287
         }
288
      }
289
   }
103 rodolico 290
   return 0; # we succeeded
96 rodolico 291
}
292
 
99 rodolico 293
&readConfig() or die "Could not read config file: $!\n";
119 rodolico 294
# if debug set, be sure testing is set also, so no actions are taken
295
$config->{'TESTING'} = 1 if $config->{'DEBUG'};
97 rodolico 296
 
103 rodolico 297
# we're pre-calculating some things so we don't do it over and over for each entry
96 rodolico 298
# grab the time once
299
my $now = time;
300
# create the string to be used for all snapshots, using $now and the template provided
301
my $snapshotName = '@' . strftime($config->{'snapshot'}->{'template'},localtime $now);
103 rodolico 302
# Create the dataset regex by joing all of the regexes defined.
96 rodolico 303
$config->{'dataset_regex'} = '(' . join( ')|(', keys %{ $config->{'datasets'} }  ) . ')' unless $config->{'dataset_regex'};
304
#print $config{'dataset_regex'} . "\n";
305
$config->{'snapshot_regex'} = '(' . $config->{'dataset_regex'} . ')@' . $config->{'snapshot'}->{'parse'};
306
#print $config->{'snapshot_regex'} . "\n\n";
307
 
308
#die Dumper( $config ) . "\n";   
309
# first, find all datasets which match our keys
310
my $dataSets = &getListing( $config, $config->{'dataset_regex'}, 'zfs list'  );
311
# and, find all snapshots that match
312
my $snapshots = &getListing( $config, $config->{'snapshot_regex'}, 'zfs list -t snapshot'  );
313
# get the date/time of the snapshots and store them in the hash
314
&parseSnapshots($snapshots, $config );
97 rodolico 315
# mergeData the snapshots into the datasets for convenience
316
&mergeData( $dataSets, $snapshots, $config );
96 rodolico 317
# Now, let's do the actual processing
318
my @commands  = &process( $dataSets, $now, $snapshotName, &period2seconds( $config->{'slop'} ) );
99 rodolico 319
#print join ( "\n", @commands ) . "\n";
96 rodolico 320
my $errors;
99 rodolico 321
print "Error: $errors\n" if $errors = &run( $config->{'TESTING'}, @commands );
96 rodolico 322
 
99 rodolico 323
# print Dumper( $dataSets );
96 rodolico 324
#print Dumper( $snapshots );
325
 
326
#print join ("\n", sort keys( %$dataSets ) ) . "\n\n";
327
#print join( "\n", sort keys( %$snapshots ) ) . "\n";
328
 
329
1;