Subversion Repositories zfs_utils

#! /usr/bin/env perl

# Simplified BSD License (FreeBSD License)

#

# Copyright (c) 2025, Daily Data Inc.

# All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are met:

#

# 1. Redistributions of source code must retain the above copyright notice, this

#    list of conditions and the following disclaimer.

#

# 2. Redistributions in binary form must reproduce the above copyright notice,

#    this list of conditions and the following disclaimer in the documentation

#    and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE

# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# sneakernet

# Script to perform sneakernet replication of ZFS datasets between two servers

# using an external transport drive.

# Uses ZFS send/receive to replicate datasets to/from the transport drive.

# Optionally uses symmetric encryption to encrypt datasets during transport.

# On the target server, can optionally use GELI to encrypt the datasets on disk.

# Requires a configuration file in YAML format next to the script.

# Author: R. W. Rodlico <rodo@dailydata.net>

# Created: December 2025

#

# Revision History:

# Version: 0.1 RWR 2025-12-10

# Development version

#

# Version: 1.0 RWR 2025-12-15

# Tested and ready for initial release

#

# Version: 1.0.1 RWR 2025-12-15

# Added verbose logging control to logMsg calls, controlled by ZFS_Utils::$verboseLoggingLevel

#

# Version: 1.1 RWR 2025-12-17

# Added filtering so only the snapshots for the current dataset are considered when

# generating replication commands.

# added test.source.status (in script directory) file for dry-run testing of source replication.

# changed default name of status file to sneakernet_target.status to avoid confusion.

# added more logging of source/target snapshots in doSourceReplication for debugging.

#

# Version: 1.1.1 RWR 2025-12-18

# Fixed makeReplicateCommands call to pass actual dataset name from config rather than config key.

# Updated documentation to clarify how makeReplicateCommands filters snapshots by parent+dataset path

# to avoid false matches with similarly-named datasets in different locations.

use strict;

use warnings;

our $VERSION = '1.1.1';

use File::Basename;

use FindBin;

use lib "$FindBin::Bin/..";

use Data::Dumper;

use ZFS_Utils qw(loadConfig shredFile logMsg makeReplicateCommands mountDriveByLabel unmountDriveByLabel mountGeli runCmd sendReport fatalError getDirectoryList cleanDirectory $logFileName $displayLogsOnConsole $verboseLoggingLevel);

use Getopt::Long qw(GetOptions);

Getopt::Long::Configure ("bundling");

my $scriptDirectory = $FindBin::RealBin;

my $scriptFullPath = "$scriptDirectory/" . $FindBin::Script;

# display all log messages on console in addition to the log file

$displayLogsOnConsole = 1;

my $configFileName = "$scriptFullPath.conf.yaml";

my $config = {

   'dryrun' => 0,

   'verbosity' => 1,

   # file created on source server to track last copied dataset, assuming

   # the import goes well on the target server

   'status_file' => "$scriptDirectory/sneakernet_target.status",

   'log_file' => "$scriptFullPath.log",

   #information about source server

   'source' => {

      'hostname' => '', # used to see if we are on source

      'poolname' => 'pool', # name of the ZFS pool to export

      # if set, will generate a report via email or by storing on a drive

      'report' => {

         'email' => 'tech@example.org',

         'subject' => 'AG Transport Report',

         'targetDrive' => {

            'fstype' => '', # filesystem type of the report drive

            # How often to check for the disk (seconds), message displayed every interval

            'check_interval' => 15,

            'label' => '',

            'mount_point' => '',

         }

      }

   },

   #information about target server

   'target' => {

      'hostname' => '', # used to see if we are on target

      'poolname' => 'backup', # name of the ZFS pool to import

      'shutdown_after_replication' => 0, # if set to 1, will shutdown the server after replication

      # if this is set, the dataset uses GELI, so we must decrypt and

      # mount it first

      'geli' => {

         'secureKey ' => {

            'label' => 'replica', # the GPT label of the key disk

            'fstype' => 'ufs', # filesystem type of the key disk

            'check_interval' => 15,

            'wait_timeout' => 300,

            'keyfile' => 'geli.key', # the name of the key file on the secureKey disk

         },

         'localKey' => 'e98c660cccdae1226550484d62caa2b72f60632ae0c607528aba1ac9e7bfbc9c', # hex representation of the local key part

         'target' => '/media/geli.key', # location to create the combined keyfile

         'poolname' => 'backup', # name of the ZFS pool to import

         'diskList' => [ 

            'da0',

            'da1'

            ], # list of disks to try to mount the dataset from

      },

      'report' => {

         'email' => '',

         'subject' => '',

         'targetDrive' => {

            'fstype' => 'msdos', # filesystem type of the report drive

            'label' => 'sneakernet',

            'mount_point' => '',

         }

      }

   },

   'transport' => {

      # this is the GPT label of the sneakernet disk

      'label' => 'sneakernet',

      # this is the file system type. Not needed if ufs

      'fstype' => 'ufs',

      # where we want to mount it

      'mount_point' => '/mnt/sneakernet',

      # amount of time to wait for the disk to appear

      'timeout' => 600,

      # How often to check for the disk (seconds), message displayed every interval

      'check_interval' => 15,

      # if set, all files will be encrypted with this key/IV during transport

      'encryption' => {

         'key'    => '', # openssl rand 32 | xxd -p | tr -d '\n' > test.key

         'IV'     => '00000000000000000000000000000000',

      },

   },

   'datasets' => {

      'dataset1' => {

         'source' => 'pool', # the parent of the dataset on the source

         'target' => 'backup', # the parent of the dataset on the target

         'dataset' => 'dataset1', # the dataset name

      },

      'files_share'  => {

         'source' => 'pool',

         'target' => 'backup',

         'dataset' => 'files_share',

      },

   }

};

## Read the status file and return its lines as an ARRAYREF.

##

## Arguments:

##   $filename - path to the status file (string)

##

## Behavior:

##   - If the file exists and is readable, reads all lines, chomps newlines and returns an ARRAYREF

##     containing the lines (each line generally holds a fully qualified snapshot name).

##   - If the file does not exist or cannot be opened, logs a message and returns an empty ARRAYREF.

##

## Returns: ARRAYREF of lines (possibly empty).

sub getStatusFile {

   my $filename = shift;

   # read in history/status file

   my @lines = ();

   if ( -e $filename && open my $fh, '<', $filename ) {

      chomp( @lines = <$fh> );

      close $fh;

      logMsg("Read status file '$filename' with contents:\n" . join( "\n", @lines ) . "\n") if $verboseLoggingLevel >= 3;

   } else {

      logMsg("Error: could not read status file '$filename', assuming a fresh start: $!") if $verboseLoggingLevel >= 2;

   }

   return \@lines;

}

## Write the status list to disk safely.

##

## Arguments:

##   $filename   - path to the status file to write

##   $statusList - ARRAYREF of lines to write into the file

##

## Behavior:

##   - If an existing file is present, renames it to `$filename.bak` as a simple backup.

##   - Writes the provided lines to `$filename` (one per line).

##   - Logs the written contents. Dies on failure to backup or write the file.

sub writeStatusFile {

   my ( $filename, $statusList ) = @_;

   # backup existing status file

   if ( -e $filename ) {

      rename( $filename, "$filename.bak" ) 

         or fatalError("Error: could not backup existing status file '$filename': $!");

   }

   # write new status file

   if ( open my $fh, '>', $filename ) {

      foreach my $line ( @$statusList ) {

         print $fh "$line\n";

      }

      close $fh;

      logMsg("Wrote status file '$filename' with contents:\n" . join( "\n", @$statusList ) . "\n") if $verboseLoggingLevel >= 3;

   } else {

      fatalError("Error: could not write status file '$filename': $!");

   }

}

## Convert a path-like dataset name into a filename-safe string.

##

## Examples:

##   'pool/fs/sub' => 'pool.fs.sub' (default)

##

## Arguments:

##   $string       - input string to convert

##   $delimiter    - input delimiter to split on (default: '/')

##   $substitution - output separator to join with (default: '.')

##

## Returns: a joined string suitable for use as a filename.

sub dirnameToFileName {

   my ( $string, $delimiter, $substitution ) = @_;

   $delimiter //= '/';

   $substitution //= '.';

   my @parts = split( /\Q$delimiter\E/, $string );

   return join( $substitution, @parts );

}

## Perform replication for all configured datasets on the source server.

##

## Arguments:

##   $config     - configuration HASHREF (loaded from YAML). Must contain `datasets` and `transport` entries.

##   $statusList - ARRAYREF of previously replicated snapshot full names (used to compute incremental sends)

##

## Behavior:

##   - Iterates over datasets defined in `$config->{datasets}`.

##   - For each dataset, enumerates available snapshots on the source and calls

##     `makeReplicateCommands` (from ZFS_Utils) to produce zfs send commands.

##   - `makeReplicateCommands` filters snapshots by matching parent+dataset path (e.g., ^pool/dataset(?:/|@))

##     to avoid false matches with similarly-named datasets, and intelligently determines whether to use

##     recursive vs per-filesystem sends and incremental vs full sends based on snapshot availability.

##   - Commands are optionally piped through `openssl enc` when `transport.encryption.key` is set.

##   - The output is written to files on the transport mount point (one file per dataset snapshot set).

##   - Respects `$config->{dryrun}`: no commands are executed when dryrun is enabled.

##

## Returns: ARRAYREF `$newStatus` containing updated status lines (latest snapshots per dataset).

sub doSourceReplication {

   my ($config, $targetList) = @_;

   my $newStatus = [];

   foreach my $dataset ( sort keys %{$config->{datasets}} ) {

      logMsg("Processing dataset '$dataset'") if $verboseLoggingLevel >= 1;

      # get list of all snapshots on dataset

      my $sourceList;

      if ( -e "$scriptDirectory/test.source.status" && $config->{dryrun} ) {

         logMsg("Using test.source.status file for source snapshots in dry-run mode") if $verboseLoggingLevel >= 2;

         $sourceList = getStatusFile( "$scriptDirectory/test.source.status" );

      } else {

         $sourceList = [ runCmd( "zfs list -rt snap -H -o name $config->{datasets}->{$dataset}->{source}/$config->{datasets}->{$dataset}->{dataset}" ) ];

      }

      if ($verboseLoggingLevel >= 5) {

         logMsg("Source snapshots for dataset '$dataset': " . join( ', ', sort @$sourceList ) );

         logMsg("Target snapshots for dataset '$dataset': " . join( ', ', sort @$targetList ) );

      }

      # process dataset here using makeReplicateCommands from ZFS_Utils

      # $dataset here is the config key; the actual dataset name is in $config->{datasets}->{$dataset}->{dataset}

      my $commands = makeReplicateCommands( 

                        $sourceList,

                        $targetList,

                        $config->{datasets}->{$dataset}->{dataset},

                        $config->{datasets}->{$dataset}->{source},

                        $config->{datasets}->{$dataset}->{target},

                        $newStatus

                     );

      if ( %$commands ) {

         foreach my $cmd ( keys %$commands ) {

            my $command = $commands->{$cmd};

            my $outputFile = $cmd;

            my $outfile = dirnameToFileName( $cmd );

            $command .= " | openssl enc -aes-256-cbc -K $config->{transport}->{encryption}->{key} -iv $config->{transport}->{encryption}->{IV} " if $config->{transport}->{encryption}->{key};

            $command .= " > $config->{transport}->{mount_point}/$outfile";

            logMsg("Running command: $command") if $verboseLoggingLevel >= 2;

            runCmd(  $command  ) unless $config->{dryrun};

         }

      } else {

         logMsg( "Nothing to do for $dataset" ) if $verboseLoggingLevel >= 1;

      }

   }

   return $newStatus;

}

## Perform cleanup and final reporting after replication.

##

## Arguments:

##   $config  - configuration HASHREF (required)

##   $message - OPTIONAL message to include in the report

##

## Behavior:

##   - Logs disk usage for the transport mount and zpool list for diagnostics.

##   - Ensures the report subject and message are populated, then attempts to unmount

##     the transport drive and send the report (via `sendReport`).

##   - If `shutdown_after_replication` is set in the running role's config, attempts

##     to shut down the machine (honors `$config->{dryrun}`).

sub cleanup{

   my ( $config, $message ) = @_;

   # add disk space utilization information on transport to the log

   logMsg( "Disk space utilization on transport disk:\n" . runCmd( "df -h $config->{transport}->{mount_point}" ) . "\n" )

      if -d $config->{transport}->{mount_point} && $verboseLoggingLevel >= 1;

   # add information about the server (zpools) to the log

   my $servername = `hostname -s`;

   chomp $servername;

   logMsg( "Zpools on server $servername:\n" . join( "\n", runCmd( "zpool list" ) ) . "\n" ) if `which zpool` && $verboseLoggingLevel >= 1;

   $config->{$config->{runningAs}}->{report}->{subject} //= "Replication Report for $config->{runningAs} server $servername";

   $message //= "Replication completed on $config->{runningAs} server $servername.";

   # unmount the sneakernet drive

   unmountDriveByLabel( $config->{transport} ) unless $config->{dryrun};

   sendReport( $config->{$config->{runningAs}}->{report}, $message, $config->{log_file} );

   # If they have requested shutdown, do it now

   if ( $config->{$config->{runningAs}}->{shutdown_after_replication} ) {

      logMsg( "Shutting down target server as per configuration" ) if $verboseLoggingLevel >= 0;

      runCmd( "shutdown -p now" ) unless $config->{dryrun};

   }

}

## Update the target zfs datasets from files on the transport drive.

##

## Arguments:

##   $config - configuration HASHREF containing `transport` and `datasets` entries.

##

## Behavior:

##   - Reads all regular files from the transport mount point (via `getDirectoryList`).

##   - For each file, determines the intended target dataset based on the filename and

##     the `datasets` mapping in the config, optionally decrypts via `openssl enc -d`,

##     and pipes the stream into `zfs receive -F` to update the target dataset.

##   - Uses `runCmd` to execute the receive commands and logs the executed command string.

sub updateTarget {

   my $config = shift;

   my $files = getDirectoryList( $config->{transport}->{mount_point});

   foreach my $filename ( @$files ) {

      my $targetDataset = basename( $filename );

      my ($dataset) = split( /\Q\.\E/, $targetDataset ); # grab only the first element of a string which has internal delimiters

      $targetDataset = $config->{datasets}->{$dataset}->{target} . '/' . dirnameToFileName( $targetDataset, '.', '/' );

      my $command = "cat $filename";

      $command .= " | openssl enc -aes-256-cbc -d -K $config->{transport}->{encryption}->{key} -iv $config->{transport}->{encryption}->{IV} " if $config->{transport}->{encryption}->{key};

      $command .= " | zfs receive -F $targetDataset";

      logMsg( $command ) if $verboseLoggingLevel >= 2;

      runCmd( $command );

   }

}

##################### main program starts here #####################

# Example to create a random key for encryption/decryption:

# generate a random key with

# openssl rand 32 | xxd -p | tr -d '\n' > test.key

# If a YAML config file exists next to the script, load and merge it

$config = loadConfig($configFileName, $config );

exit 1 unless keys %$config;

# parse CLI options

GetOptions( $config,

   'dryrun|n',

   'verbosity|v+',

   'version|V',

   'help|h',

) or do { print "Invalid options\n"; exit 2 };

if (defined ($config->{help})) {

   print "Usage: $FindBin::Script [--dryrun] [--verbosity] [--help]\n";

   print "  --dryrun, -n   Run in dry-run mode (no writes)\n";

   print "  --verbosity, -v  Run in verbose mode (more v's mean more verbose)\n";

   print "  --version, -V  Display version number\n";

   exit 0;

} elsif (defined $config->{version}) {

   print "$FindBin::Script v$VERSION\n";

   exit 0;

}

# set some defaults in library from config

$verboseLoggingLevel = $config->{verbosity} // 0;

# status file path

$config->{'status_file'} //= "$scriptFullPath.status";

# set log file name for sub logMsg in ZFS_Utils, and remove the old log if it exists

# Log file is only valid for one run

$logFileName = $config->{'log_file'} //= "$scriptFullPath.log";

# log only for one run

unlink ( $logFileName ) if -f $logFileName;

fatalError( "Invalid config file: missing source and/or target server", $config, \&cleanup )

    unless (defined $config->{source} && defined $config->{target});

my $servername = `hostname -s`;

chomp $servername;

$config->{runningAs} = $servername eq $config->{source}->{hostname} ? 'source' :

                $servername eq $config->{target}->{hostname} ? 'target' : 'unknown';

# mount the transport drive, fatal error if we can not find it

fatalError( "Unable to mount tranport drive with label $config->{transport}->{label}", $config, \&cleanup )

   unless $config->{dryrun} or $config->{transport}->{mount_point} =  mountDriveByLabel( $config->{transport} );

# main program logic

if ( $config->{runningAs} eq 'source' ) {

    logMsg "Running as source server" if $verboseLoggingLevel >= 1;

    # remove all files from transport disk, but leave all subdirectories alone

   fatalError( "Failed to clean transport directory $config->{transport}->{mount_point}", $config, \&cleanup )

      unless $config->{dryrun} or cleanDirectory( $config->{transport}->{mount_point} );

    my $statusList = getStatusFile($config->{status_file});

    $statusList = doSourceReplication($config, $statusList); 

    writeStatusFile($config->{status_file}, $statusList) unless $config->{dryrun};

} elsif ( $config->{runningAs} eq 'target' ) {

    logMsg "Running as target server" if $verboseLoggingLevel >= 1;

    mountGeli( $config->{target}->{geli} ) if ( defined $config->{target}->{geli} );

    umountDiskByLabel( $config->{target}->{geli}->{secureKey} )

       unless $config->{target}->{geli}->{secureKey}->{label} eq $config->{transport}->{label};

    print "Please insert device labeled REPORT\n" if $config->{target}->{report}->{targetDrive}->{label};

    updateTarget( $config );

} else {

    fatalError( "This server ($servername) is neither source nor target server as per config\n" );

}

cleanup( $config );

1;