Rev 42 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed
#! /usr/bin/env perl
# Simplified BSD License (FreeBSD License)
#
# Copyright (c) 2025, Daily Data Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# sneakernet.pl
# Script to perform sneakernet replication of ZFS datasets between two servers
# using an external transport drive.
# Uses ZFS send/receive to replicate datasets to/from the transport drive.
# Optionally uses symmetric encryption to encrypt datasets during transport.
# On the target server, can optionally use GELI to encrypt the datasets on disk.
# Requires a configuration file in YAML format next to the script.
# Author: R. W. Rodlico <rodo@dailydata.net>
# Created: December 2025
# Revision History:
# Version: 0.1 2025-12-10 Initial version
use strict;
use warnings;
our $VERSION = '0.1';
use FindBin;
use lib "$FindBin::Bin/..";
use Data::Dumper;
use ZFS_Utils qw(loadConfig shredFile logMsg makeReplicateCommands mountDriveByLabel unmountDriveByLabel mountGeli runCmd sendReport fatalError cleanDirectory $logFileName $displayLogsOnConsole);
my $scriptDirectory = $FindBin::RealBin;
my $scriptFullPath = "$scriptDirectory/" . $FindBin::Script;
# if set, will not actually write files to disk
my $DEBUG = 1;
# display all log messages on console in addition to the log file
$displayLogsOnConsole = 1;
my $configFileName = "$scriptFullPath.conf.yaml";
my $config = {
# file created on source server to track last copyed dataset
'status_file' => "$scriptFullPath.status",
'log_file' => "$scriptFullPath.log",
#information about source server
'source' => {
'hostname' => '', # used to see if we are on source
'poolname' => 'pool', # name of the ZFS pool to export
# if set, will generate a report via email or by storing on a drive
'report' => {
'email' => 'tech@example.org',
'subject' => 'AG Transport Report',
'targetDrive' => {
'fstype' => '', # filesystem type of the report drive
# How often to check for the disk (seconds), message displayed every interval
'check_interval' => 15,
'label' => '',
'mount_point' => '',
}
}
},
#information about target server
'target' => {
'hostname' => '', # used to see if we are on target
'poolname' => 'backup', # name of the ZFS pool to import
'shutdown_after_replication' => 0, # if set to 1, will shutdown the server after replication
# if this is set, the dataset uses GELI, so we must decrypt and
# mount it first
'geli' => {
'secureKey ' => {
'label' => 'replica', # the GPT label of the key disk
'fstype' => 'ufs', # filesystem type of the key disk
'check_interval' => 15,
'wait_timeout' => 300,
'keyfile' => 'geli.key', # the name of the key file on the secureKey disk
},
'localKey' => 'e98c660cccdae1226550484d62caa2b72f60632ae0c607528aba1ac9e7bfbc9c', # hex representation of the local key part
'target' => '/media/geli.key', # location to create the combined keyfile
'poolname' => 'backup', # name of the ZFS pool to import
'diskList' => [
'da0',
'da1'
], # list of disks to try to mount the dataset from
},
'report' => {
'email' => '',
'subject' => '',
'targetDrive' => {
'fstype' => 'msdos', # filesystem type of the report drive
'label' => 'sneakernet',
'mount_point' => '',
}
}
},
'transport' => {
# this is the GPT label of the sneakernet disk
'label' => 'sneakernet',
# this is the file system type. Not needed if ufs
'fstype' => 'ufs',
# where we want to mount it
'mount_point' => '/mnt/sneakernet',
# amount of time to wait for the disk to appear
'timeout' => 600,
# How often to check for the disk (seconds), message displayed every interval
'check_interval' => 15,
# if set, all files will be encrypted with this key/IV during transport
'encryption' => {
'key' => '', # openssl rand 32 | xxd -p | tr -d '\n' > test.key
'IV' => '00000000000000000000000000000000',
},
},
'datasets' => {
'dataset1' => {
'source' => 'pool', # the parent of the dataset on the source
'target' => 'backup', # the parent of the dataset on the target
'dataset' => 'dataset1', # the dataset name
},
'files_share' => {
'source' => 'pool',
'target' => 'backup',
'dataset' => 'files_share',
},
}
};
# read the status file and return as list. If the file doesn't exits, returns an empty list
sub getStatusFile {
my $filename = shift;
# read in history/status file
my @lines = ();
if ( -e $filename && open my $fh, '<', $filename ) {
chomp( @lines = <$fh> );
close $fh;
logMsg("Read status file '$filename' with contents:\n" . join( "\n", @lines ) . "\n");
} else {
logMsg("Error: could not read status file '$filename', assuming a fresh start: $!");
}
return \@lines;
}
# write the status list to file
sub writeStatusFile {
my ( $filename, $statusList ) = @_;
# backup existing status file
if ( -e $filename ) {
rename( $filename, "$filename.bak" ) or do {
logMsg("Error: could not backup existing status file '$filename': $!");
die;
};
}
# write new status file
if ( open my $fh, '>', $filename ) {
foreach my $line ( @$statusList ) {
print $fh "$line\n";
}
close $fh;
logMsg("Wrote status file '$filename' with contents:\n" . join( "\n", @$statusList ) . "\n");
} else {
logMsg("Error: could not write status file '$filename': $!");
die;
}
}
# simple sub to take root/dataset/datset/dataset and turn it into
# dataset.dataset.dataset
sub replaceSlashWithDot {
my $string = shift;
my @parts = split( "/", $string );
shift @parts;
return join( '.', @parts );
}
# perform replication on source server
# $config - configuration hashref
# $statusList - list of last snapshots replicated for each dataset in previous replications
# return new status list after replication containing updated last snapshots
# this script will actually replicate the datasets to the sneakernet disk
sub doSourceReplication {
my ($config, $statusList) = @_;
my $newStatus = [];
foreach my $dataset ( sort keys %{$config->{datasets}} ) {
logMsg("Processing dataset '$dataset'");
# get list of all snapshots on dataset
my $root = $config->{datasets}->{$dataset}->{source} . '/' . $config->{datasets}->{$dataset}->{dataset};
my $sourceList = [ runCmd( "zfs list -rt snap -H -o name $root" ) ];
# remove the parent part, leave the dataset itself
$sourceList =~ s|$config->{datasets}->{$dataset}->{source}/||;
# process dataset here
my $commands = makeReplicateCommands( $sourceList, $statusList, $newStatus );
if ( %$commands ) {
foreach my $cmd ( keys %$commands ) {
my $command = $commands->{$cmd};
my $outputFile = $cmd;
$outputFile =~ s/^$root//;
$outputFile = replaceSlashWithDot($outputFile);
#$command .= " | openssl enc -aes-256-cbc -K $config->{transport}->{encryption}->{key} -iv $config->{transport}->{encryption}->{IV} " if $config->{transport}->{encryption}->{key};
$command .= " > $config->{transport}->{mount_point}/" . $outputFile;
logMsg("Running command: $command");
runCmd( $command ) unless $DEBUG;
}
} else {
logMsg( "Nothing to do for $dataset" );
}
}
return $newStatus;
}
# perform cleanup actions
# $config - configuration hashref
# $message - optional message to include in the report
#
sub cleanup{
my ( $config, $message ) = @_;
# add disk space utilization information on transport to the log
logMsg( "Disk space utilization on transport disk:\n" . runCmd( "df -h $config->{transport}->{mount_point}" ) . "\n" );
# add information about the server (zpools) to the log
my $servername = `hostname -s`;
chomp $servername;
logMsg( "Zpools on server $servername:\n" . join( "\n", runCmd( "zpool list" ) ) . "\n" );
$config->{$config->{runningAs}}->{report}->{subject} //= "Replication Report for $config->{runningAs} server $servername";
$message //= "Replication completed on $config->{runningAs} server $servername.";
# unmount the sneakernet drive
unmountDriveByLabel( $config->{transport} );
sendReport( $config->{$config->{runningAs}}->{report}, $message, $config->{log_file} );
# If they have requested shutdown, do it now
if ( $config->{$config->{runningAs}}->{shutdown_after_replication} ) {
logMsg( "Shutting down target server as per configuration" );
runCmd( "shutdown -p now" ) unless $DEBUG;
}
}
# update the target datasets from the files on the transport drive
sub updateTarget {
my $config = shift;
my $files = getDirectoryList( $config->{transport}->{mount_point});
foreach my $filename ( @$files ) {
my $command = "cat $config->{output} | openssl enc -aes-256-cbc -d -K $config->{key} -iv $config->{IV}";
}
}
##################### main program starts here #####################
# Example to create a random key for encryption/decryption:
# generate a random key with
# openssl rand 32 | xxd -p | tr -d '\n' > test.key
# If a YAML config file exists next to the script, load and merge it
$config = loadConfig($configFileName, $config );
exit 1 unless keys %$config;
# set some defaults
$config->{'status_file'} //= "$scriptFullPath.status";
# set log file name for sub logMsg in ZFS_Utils, and remove the old log if it exists
# Log file is only valid for one run
$logFileName = $config->{'log_file'} //= "$scriptFullPath.log";
# log only for one run
unlink ( $logFileName ) if -f $logFileName;
fatalError( "Invalid config file: missing source and/or target server", $config, \&cleanup )
unless (defined $config->{source} && defined $config->{target});
my $servername = `hostname -s`;
chomp $servername;
$config->{runningAs} = $servername eq $config->{source}->{hostname} ? 'source' :
$servername eq $config->{target}->{hostname} ? 'target' : 'unknown';
#cleanup( $config, "Testing" );
# mount the transport drive, fatal error if we can not find it
fatalError( "Unable to mount tranport drive with label $config->{transport}->{disk_label}", $config, \&cleanup )
unless $config->{transport}->{mount_point} = mountDriveByLabel( $config->{transport} );
# mail program logic
if ( $config->{runningAs} eq 'source' ) {
logMsg "Running as source server";
# remove all files from transport disk, but leave all subdirectories alone
fatalError( "Failed to clean transport directory $config->{transport}->{mount_point}", $config, \&cleanup )
unless cleanDirectory( $config->{transport}->{mount_point} );
my $statusList = getStatusFile($config->{status_file});
$statusList = doSourceReplication($config, $statusList);
writeStatusFile($config->{status_file}, $statusList);
} elsif ( $config->{runningAs} eq 'target' ) {
logMsg "Running as target server";
mountGeli( $config->{target}->{geli} ) if ( defined $config->{target}->{geli} );
updateTarget( $config );
} else {
fatalError( "This server ($servername) is neither source nor target server as per config\n" );
}
cleanup( $config );
1;