Subversion Repositories sysadmin_scripts

Rev

Go to most recent revision | Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
96 rodolico 1
#! /usr/bin/env perl
2
 
98 rodolico 3
#    snapShot: Manage ZFS snapshots
4
#    see http://wiki.linuxservertech.com for additional information
5
#    Copyright (C) 2022  R. W. Rodolico
6
#
7
#    version 1.0, 20220423
8
#       Initial Release
9
#
10
#
11
#    This program is free software: you can redistribute it and/or modify
12
#    it under the terms of the GNU General Public License as published by
13
#    the Free Software Foundation, either version 3 of the License, or
14
#    (at your option) any later version.
15
#
16
#    This program is distributed in the hope that it will be useful,
17
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
18
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19
#    GNU General Public License for more details.
20
#
21
#    You should have received a copy of the GNU General Public License
22
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
23
#
24
# Warning, this script requires YAML::Tiny and Hash::Merge Perl Modules to be installed
25
# Under Debian:  apt install libyaml-tiny-perl libhash-merge-simple-perl
26
# Under FreeBSD: cpan -i Hash::Merge::Simple YAML::Tiny
27
 
28
 
96 rodolico 29
use strict;
30
use warnings;
31
 
32
use Data::Dumper;
33
use Time::Local;
34
use POSIX qw(strftime);
35
use YAML::Tiny; # apt-get libyaml-tiny-perl under debian, BSD Systems: cpan -i YAML::Tiny
98 rodolico 36
use Hash::Merge::Simple qw/ merge clone_merge /; # apt install libhash-merge-simple-perl or cpan -i Hash::Merge::Simple
96 rodolico 37
 
38
 
39
# globals
40
my $CONFIG_FILE_NAME = 'snapShot.yaml';
41
 
42
# This will be read in from snapShot.yaml
43
my $config;
44
 
45
#
46
# find where the script is actually located as cfg should be there
47
#
48
sub getScriptLocation {
49
   use strict;
50
   use File::Spec::Functions qw(rel2abs);
51
   use File::Basename;
52
   return dirname(rel2abs($0));
53
}
54
 
55
#
56
# Read the configuration file from current location 
57
# and return it as a string
58
#
59
sub readConfig {
60
   my $scriptLocation = &getScriptLocation();
61
   if ( -e "$scriptLocation/$CONFIG_FILE_NAME" ) {
62
      my $yaml = YAML::Tiny->read( "$scriptLocation/$CONFIG_FILE_NAME" );
63
      # use clone_merge to merge conf file into $config
64
      # overwrites anything in $config if it exists in the config file
65
      $config = clone_merge( $config, $yaml->[0] );
66
      return 1;
67
   }
68
   return 0;
69
}
70
 
71
 
72
# parse one single line from the output of `zfs list [-t snapshot]`
73
sub parseListing {
74
   my ($line,$keys) = @_;
75
   chomp $line;
76
   my %values;
77
   @values{@$keys} = split( /\s+/, $line );
78
   return \%values;
79
}      
80
 
81
 
82
# this will parse the date out of the snapshots and put the values into
83
# the hash {'date'}
84
sub parseSnapshots {
85
   my ( $snapShots, $config) = @_;
86
   my $keys = $config->{'snapshot'}->{'parseFields'};
87
   foreach my $snapShot ( keys %$snapShots ) {
88
      my %temp;
89
      # run the regex, capture the output to an array, then populate the hash %temp
90
      # using the regex results as the values, and $keys as the keys
91
      @temp{@$keys} = ( $snapShot =~ m/$config->{'snapshot'}->{'parse'}/ );
92
      # while we're here, calculate the unix time (epoch). NOTE: month is 0 based
93
      $temp{'unix'} = timelocal( 0,$temp{'minute'},$temp{'hour'},$temp{'day'},$temp{'month'}-1,$temp{'year'} );
94
      # put this into our record
95
      $snapShots->{$snapShot}->{'date'} = \%temp;
96
   }
97
}
98
 
99
# run $command, then parse its output and return the results as a hashref
100
sub getListing {
101
   my ($configuration, $regex, $command )  = @_;
102
   my %dataSets;
103
 
104
   # get all datasets
105
   my @zfsList = `$command`;
106
   foreach my $thisSet ( @zfsList ) {
107
      my $temp = &parseListing( $thisSet, $configuration->{'listingKeys'} );
108
      if (  $temp->{'name'} =~ m/^($regex)$/ ) {
109
         $dataSets{$temp->{'name'}} = $temp;
110
      }
111
   }
112
   return \%dataSets;
113
}
114
 
115
# will convert something like 1 day to the number of seconds (86400) for math.
116
# month and year are approximations (30.5 day = a month, 365.2425 days is a year)
101 rodolico 117
# For month and year, use the int function to convert back to integer
96 rodolico 118
sub period2seconds {
119
   my ($count, $unit) = ( shift =~ m/\s*(\d+)\s*([a-z]+)\s*/i );
120
   $unit = lc $unit;
97 rodolico 121
   if ( $unit eq 'hour' ) {
96 rodolico 122
      $count *= 3600;
123
   } elsif ( $unit eq 'day' ) {
124
      $count *= 86400;
125
   } elsif ( $unit eq 'week' ) {
126
      $count *= 864000 * 7;
127
   } elsif ( $unit eq 'month' ) {
101 rodolico 128
      $count *= int( 864000 * 30.5 );
96 rodolico 129
   } elsif ( $unit eq 'year' ) {
101 rodolico 130
      $count *= int( 86400 * 365.2425 );
96 rodolico 131
   } else {
132
      die "Unknown units [$unit] in period2seconds\n";
133
   }
134
   return $count;
135
}
136
 
137
# Merges datasets, snapshots and some stuff from the configuration into the datasets
138
# hash
97 rodolico 139
sub mergeData {
96 rodolico 140
   my ($datasets,$snapshots,$config) = @_;
141
   my $confKeys = $config->{'datasets'};
142
   foreach my $thisDataset ( keys %$datasets ) {
143
      foreach my $conf (keys %$confKeys ) {
144
         if ( $thisDataset =~ m/^$conf$/ ) {
145
            $datasets->{$thisDataset}->{'recursive'} = $confKeys->{$conf}->{'recursive'};
146
            $datasets->{$thisDataset}->{'frequency'} = &period2seconds( $confKeys->{$conf}->{'frequency'} );
147
            $datasets->{$thisDataset}->{'retention'} = &period2seconds( $confKeys->{$conf}->{'retention'} );
148
            last;
149
         } # if
150
      } # foreach
151
      foreach my $snapshot ( keys %$snapshots ) {
152
         if ( $snapshot =~ m/^$thisDataset@/ ) { # this is a match
153
            # copy the snapshot into the dataset
154
            $datasets->{$thisDataset}->{'snapshots'}->{$snapshot} = $snapshots->{$snapshot};
155
            # track the latest snapshot
156
            $datasets->{$thisDataset}->{'lastSnap'} = $snapshots->{$snapshot}->{'date'}->{'unix'}
157
               if ! defined( $datasets->{$thisDataset}->{'lastSnap'} ) || $datasets->{$thisDataset}->{'lastSnap'} < $snapshots->{$snapshot}->{'date'}->{'unix'};
158
            # delete the snapshot
159
            delete $snapshots->{$snapshot};
160
         } # if
161
      } # foreach
162
   } # foreach
97 rodolico 163
} # sub mergeData
96 rodolico 164
 
165
sub checkRetention {
166
   my ( $retentionPeriod, $recursive, $snapshots, $now ) = @_;
167
   my @toDelete;
168
   foreach my $thisSnapshot ( keys %$snapshots ) {
169
      # print "checking $thisSnapshot\n\tNow: $now\n\tDate: $snapshots->{$thisSnapshot}->{date}->{unix}\n\tRetention: $retentionPeriod\n\n";
170
      if ( $now - $snapshots->{$thisSnapshot}->{'date'}->{'unix'} > $retentionPeriod ) {
171
         my $command = 'zfs destroy ' . ($recursive ? '-r ' : '') . $thisSnapshot;
172
         push @toDelete, $command;
173
      }
174
   }
175
   return @toDelete;
176
}   
177
 
178
sub makeSnapshot {
179
   my ( $datasetName, $recursive, $snapshotName ) = @_;
180
   return 
181
      'zfs snapshot ' . 
182
      ($recursive ? '-r ' : '') . 
183
      $datasetName . $snapshotName;
184
}
185
 
186
 
187
sub process {
188
   my ( $datasets, $now, $snapshotName, $slop ) = @_;
189
   my @actions;
190
   my @toDelete;
191
   my @toAdd;
192
 
193
   foreach my $thisDataset ( keys %$datasets ) {
194
      push( @toDelete, 
195
         &checkRetention( 
196
         $datasets->{$thisDataset}->{'retention'}, 
197
         $datasets->{$thisDataset}->{'recursive'}, 
198
         $datasets->{$thisDataset}->{'snapshots'}, 
199
         $now )
200
         );
100 rodolico 201
      if ( $datasets->{$thisDataset}->{'lastSnap'} + $datasets->{$thisDataset}->{'frequency'} + $slop > $now ) {
96 rodolico 202
         push @toAdd, &makeSnapshot( $thisDataset, $datasets->{$thisDataset}->{'recursive'}, $snapshotName )
203
      }
204
   }
205
   return ( @toDelete, @toAdd );
206
}   
207
 
208
sub run {
209
   my $testing = shift;
210
   # bail if there are no commands to run
99 rodolico 211
   return 0 unless @_;
96 rodolico 212
   if ( $testing ) { # don't do it, just dump the commands
213
      open LOG, ">/tmp/snapShot" or die "could not write to /tmp/snapShot: $!\n";
214
      print LOG join( "\n", @_ ) . "\n";
215
      close LOG;
216
   } else {
217
      my $out;
218
      return 'Not running right now';
219
      while ( my $command = shift ) {
220
         $out .= `$command` . "\n";
221
         if ( $? ) { # we had an error
222
            $out .= "Error executing command\n\t$command\n\t";
223
            if ($? == -1) {
224
                $out .= "failed to execute $command: $!";
225
            } elsif ($? & 127) {
226
                $out .= sprintf( "child died with signal %d, %s coredump", ($? & 127),  ($? & 128) ? 'with' : 'without' );
227
            } else {
228
                $out .= sprintf( "child exited with value %d", $? >> 8 );
229
            }
230
            $out .= "\n";
231
            return $out;
232
         }
233
      }
234
   }
235
   return 0;
236
}
237
 
99 rodolico 238
&readConfig() or die "Could not read config file: $!\n";
97 rodolico 239
 
96 rodolico 240
# grab the time once
241
my $now = time;
242
# create the string to be used for all snapshots, using $now and the template provided
243
my $snapshotName = '@' . strftime($config->{'snapshot'}->{'template'},localtime $now);
244
# Create the dataset regex for later use 
245
$config->{'dataset_regex'} = '(' . join( ')|(', keys %{ $config->{'datasets'} }  ) . ')' unless $config->{'dataset_regex'};
246
#print $config{'dataset_regex'} . "\n";
247
$config->{'snapshot_regex'} = '(' . $config->{'dataset_regex'} . ')@' . $config->{'snapshot'}->{'parse'};
248
#print $config->{'snapshot_regex'} . "\n\n";
249
 
250
#die Dumper( $config ) . "\n";   
251
# first, find all datasets which match our keys
252
my $dataSets = &getListing( $config, $config->{'dataset_regex'}, 'zfs list'  );
253
# and, find all snapshots that match
254
my $snapshots = &getListing( $config, $config->{'snapshot_regex'}, 'zfs list -t snapshot'  );
255
# get the date/time of the snapshots and store them in the hash
256
&parseSnapshots($snapshots, $config );
97 rodolico 257
# mergeData the snapshots into the datasets for convenience
258
&mergeData( $dataSets, $snapshots, $config );
96 rodolico 259
# Now, let's do the actual processing
260
my @commands  = &process( $dataSets, $now, $snapshotName, &period2seconds( $config->{'slop'} ) );
99 rodolico 261
#print join ( "\n", @commands ) . "\n";
96 rodolico 262
my $errors;
99 rodolico 263
print "Error: $errors\n" if $errors = &run( $config->{'TESTING'}, @commands );
96 rodolico 264
 
265
 
99 rodolico 266
# print Dumper( $dataSets );
96 rodolico 267
#print Dumper( $snapshots );
268
 
269
#print join ("\n", sort keys( %$dataSets ) ) . "\n\n";
270
#print join( "\n", sort keys( %$snapshots ) ) . "\n";
271
 
272
1;