Rev 35 | Rev 42 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed
#! /usr/bin/env perl
# Simplified BSD License (FreeBSD License)
#
# Copyright (c) 2025, Daily Data Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# sneakernet.pl
# Script to perform sneakernet replication of ZFS datasets between two servers
# using an external transport drive.
# Uses ZFS send/receive to replicate datasets to/from the transport drive.
# Optionally uses symmetric encryption to encrypt datasets during transport.
# On the target server, can optionally use GELI to encrypt the datasets on disk.
# Requires a configuration file in YAML format next to the script.
# Author: R. W. Rodlico <rodo@dailydata.net>
# Created: December 2025
# Revision History:
# Version: 0.1 2025-12-10 Initial version
use strict;
use warnings;
our $VERSION = '0.1';
use FindBin;
use lib "$FindBin::Bin/..";
use Data::Dumper;
use ZFS_Utils qw(loadConfig shredFile logMsg makeReplicateCommands mountDriveByLabel mountGeli runCmd sendReport $logFileName $displayLogsOnConsole);
my $scriptDirectory = $FindBin::RealBin;
my $scriptFullPath = "$scriptDirectory/" . $FindBin::Script;
# if set, will not actually write files to disk
my $DEBUG = 1;
# display all log messages on console in addition to the log file
$displayLogsOnConsole = 1;
my $configFileName = "$scriptFullPath.conf.yaml";
my $config = {
# file created on source server to track last copyed dataset
'status_file' => "$scriptFullPath.status",
'log_file' => "$scriptFullPath.log",
#information about source server
'source' => {
'hostname' => '', # used to see if we are on source
'poolname' => 'pool', # name of the ZFS pool to export
# if set, will generate a report via email or by storing on a drive
'report' => {
'email' => 'tech@example.org',
'subject' => 'AG Transport Report',
'targetDrive' => {
'fstype' => '', # filesystem type of the report drive
# How often to check for the disk (seconds), message displayed every interval
'check_interval' => 15,
'label' => '',
'mount_point' => '',
}
}
},
#information about target server
'target' => {
'hostname' => '', # used to see if we are on target
'poolname' => 'backup', # name of the ZFS pool to import
# if this is set, the dataset uses GELI, so we must decrypt and
# mount it first
'geli' => {
'secureKey ' => {
'label' => 'replica', # the GPT label of the key disk
'fstype' => 'ufs', # filesystem type of the key disk
'check_interval' => 15,
'wait_timeout' => 300,
'keyfile' => 'geli.key', # the name of the key file on the secureKey disk
},
'localKey' => 'e98c660cccdae1226550484d62caa2b72f60632ae0c607528aba1ac9e7bfbc9c', # hex representation of the local key part
'target' => '/media/geli.key', # location to create the combined keyfile
'poolname' => 'backup', # name of the ZFS pool to import
'diskList' => [
'da0',
'da1'
], # list of disks to try to mount the dataset from
},
'report' => {
'email' => '',
'subject' => '',
'targetDrive' => {
'fstype' => 'msdos', # filesystem type of the report drive
'label' => 'sneakernet',
'mount_point' => '',
}
}
},
'transport' => {
# this is the GPT label of the sneakernet disk
'label' => 'sneakernet',
# this is the file system type. Not needed if ufs
'fstype' => 'ufs',
# where we want to mount it
'mount_point' => '/mnt/sneakernet',
# amount of time to wait for the disk to appear
'timeout' => 600,
# How often to check for the disk (seconds), message displayed every interval
'check_interval' => 15,
# if set, all files will be encrypted with this key/IV during transport
'encryption' => {
'key' => '', # openssl rand 32 | xxd -p | tr -d '\n' > test.key
'IV' => '00000000000000000000000000000000',
},
},
'datasets' => {
'dataset1' => {
'source' => 'pool/dataset1',
'target' => 'backup/dataset1',
'filename' => 'dataset1'
},
'files_share' => {
'source' => 'pool/files_share',
'target' => 'backup/files_share',
'filename' => 'files_share'
},
}
};
# read the status file and return as list. If the file doesn't exits, returns an empty list
sub getStatusFile {
my $filename = shift;
# read in history/status file
my @lines = ();
if ( -e $filename && open my $fh, '<', $filename ) {
chomp( @lines = <$fh> );
close $fh;
logMsg("Read status file '$filename' with contents:\n" . join( "\n", @lines ) . "\n");
} else {
logMsg("Error: could not read status file '$filename', assuming a fresh start: $!");
}
return \@lines;
}
# write the status list to file
sub writeStatusFile {
my ( $filename, $statusList ) = @_;
# backup existing status file
if ( -e $filename ) {
rename( $filename, "$filename.bak" ) or do {
logMsg("Error: could not backup existing status file '$filename': $!");
die;
};
}
# write new status file
if ( open my $fh, '>', $filename ) {
foreach my $line ( @$statusList ) {
print $fh "$line\n";
}
close $fh;
logMsg("Wrote status file '$filename' with contents:\n" . join( "\n", @$statusList ) . "\n");
} else {
logMsg("Error: could not write status file '$filename': $!");
die;
}
}
# simple sub to take root/dataset/datset/dataset and turn it into
# dataset.dataset.dataset
sub replaceSlashWithDot {
my $string = shift;
my @parts = split( "/", $string );
shift @parts;
return join( '.', @parts );
}
# perform replication on source server
# $config - configuration hashref
# $statusList - list of last snapshots replicated for each dataset in previous replications
# return new status list after replication containing updated last snapshots
# this script will actually replicate the datasets to the sneakernet disk
sub doSourceReplication {
my ($config, $statusList) = @_;
my $newStatus = [];
foreach my $dataset ( sort keys %{$config->{datasets}} ) {
logMsg("Processing dataset '$dataset'");
# get list of all snapshots on dataset
my $sourceList = [ runCmd( "zfs list -rt snap -H -o name $config->{datasets}->{$dataset}->{source} " ) ];
# process dataset here
my $commands = makeReplicateCommands($sourceList, $statusList, $newStatus );
if ( %$commands ) {
foreach my $cmd ( keys %$commands ) {
my $command = $commands->{$cmd};
$command .= " | openssl enc -aes-256-cbc -K $config->{transport}->{encryption}->{key} -iv $config->{transport}->{encryption}->{IV} " if $config->{transport}->{encryption}->{key};
$command .= " > $config->{transport}->{mount_point}/" . replaceSlashWithDot($cmd);
logMsg("Running command: $command");
runCmd( $command ) unless $DEBUG;
}
} else {
logMsg( "Nothing to do for $dataset" );
}
}
return $newStatus;
}
# clean all files from a directory, but not any subdirectories
sub cleanDirectory {
my $dirname = shift;
logMsg( "Cleaning up $dirname of all files" );
# clean up a directory
opendir( my $dh, $dirname ) || fatalError( "Can not open $dirname: #!" );
# get all file names, but leave directories alone
my @files = map{ $dirname . "/$_" } grep { -f "$dirname/$_" } readdir($dh);
closedir $dh;
foreach my $file (@files) {
unlink $file or warn "Could not unlink $file: #!\n";
}
}
# how to handle a fatal error
sub fatalError {
my $message = shift;
logMsg( $message );
die;
}
##################### main program starts here #####################
# Example to create a random key for encryption/decryption:
# generate a random key with
# openssl rand 32 | xxd -p | tr -d '\n' > test.key
# If a YAML config file exists next to the script, load and merge it
$config = loadConfig($configFileName, $config );
# set some defaults
$config->{'status_file'} //= "$scriptFullPath.status";
# set log file name for sub logMsg in ZFS_Utils, and remove the old log if it exists
# Log file is only valid for one run
$logFileName = $config->{'log_file'} //= "$scriptFullPath.log";
# log only for one run
unlink ( $logFileName ) if -f $logFileName;
fatalError( "Invalid config file: missing source and/or target server" )
unless (defined $config->{source} && defined $config->{target});
# mount the transport drive, fatal error if we can not find it
fatalError( "Unable to mount tranport drive with label $config->{transport}->{disk_label}" )
unless $config->{transport}->{mount_point} = mountDriveByLabel( $config->{transport} );
my $servername = `hostname -s`;
chomp $servername;
my $runningAs = $servername eq $config->{source}->{hostname} ? 'source' :
$servername eq $config->{target}->{hostname} ? 'target' : 'unknown';
if ( $runningAs eq 'source' ) {
logMsg "Running as source server";
# remove all files from transport disk, but leave all subdirectories alone
cleanDirectory( $config->{transport}->{mount_point} );
my $statusList = getStatusFile($config->{status_file});
$statusList = doSourceReplication($config, $statusList);
writeStatusFile($config->{status_file}, $statusList);
} elsif ( $runningAs eq 'target' ) {
logMsg "Running as target server";
mountGeli( $config->{target}->{geli} ) if ( defined $config->{target}->{geli} );
} else {
fatalError( "This server ($servername) is neither source nor target server as per config\n" );
}
# add disk space utilization information on transport to the log
logMsg( "Disk space utilization on transport disk:\n" . runCmd( "df -h $config->{transport}->{mount_point}" ) . "\n" );
# add information about the server (zpools) to the log
logMsg( "Zpools on server $servername:\n" . join( "\n", runCmd( "zpool list" ) ) . "\n" );
# unmount the sneakernet drive
`umount $config->{transport}->{mount_point}`;
# and remove the directory
rmdir $config->{transport}->{mount_point};
1;
#`cat $config->{input} | openssl enc -aes-256-cbc -K $config->{key} -iv $config->{IV} > $config->{output}`;
# this will decrypt $config->{output} to stdout
#`cat $config->{output} | openssl enc -aes-256-cbc -d -K $config->{key} -iv $config->{IV} > test.out`;
Generated by GNU Enscript 1.6.5.90.