Subversion Repositories zfs_utils

#! /usr/bin/env perl

# Simplified BSD License (FreeBSD License)

#

# Copyright (c) 2025, Daily Data Inc.

# All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are met:

#

# 1. Redistributions of source code must retain the above copyright notice, this

#    list of conditions and the following disclaimer.

#

# 2. Redistributions in binary form must reproduce the above copyright notice,

#    this list of conditions and the following disclaimer in the documentation

#    and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE

# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# sneakernet.pl

# Script to perform sneakernet replication of ZFS datasets between two servers

# using an external transport drive.

# Uses ZFS send/receive to replicate datasets to/from the transport drive.

# Optionally uses symmetric encryption to encrypt datasets during transport.

# On the target server, can optionally use GELI to encrypt the datasets on disk.

# Requires a configuration file in YAML format next to the script.

# Author: R. W. Rodlico <rodo@dailydata.net>

# Created: December 2025

# Revision History:

# Version: 0.1 2025-12-10 Initial version

use strict;

use warnings;

our $VERSION = '0.1';

use FindBin;

use lib "$FindBin::Bin/..";

use Data::Dumper;

use ZFS_Utils qw(loadConfig shredFile logMsg makeReplicateCommands mountDriveByLabel mountGeli runCmd sendReport $logFileName $displayLogsOnConsole);

my $scriptDirectory = $FindBin::RealBin;

my $scriptFullPath = "$scriptDirectory/" . $FindBin::Script;

# if set, will not actually write files to disk

my $DEBUG = 0;

# display all log messages on console in addition to the log file

$displayLogsOnConsole = 1;

my $configFileName = "$scriptFullPath.conf.yaml";

my $config = {

   # file created on source server to track last copyed dataset

   'status_file' => "$scriptFullPath.status",

   'log_file' => "$scriptFullPath.log",

   #information about source server

   'source_server' => {

      'hostname' => '', # used to see if we are on source

      'poolname' => 'pool', # name of the ZFS pool to export

      # if set, will generate a report via email or by storing on a drive

      'report' => {

         'email' => 'tech@example.org',

         'subject' => 'AG Transport Report',

         'targetDrive' => {

            'fstype' => '', # filesystem type of the report drive

            # How often to check for the disk (seconds), message displayed every interval

            'check_interval' => 15,

            'label' => '',

            'mount_point' => '',

         }

      }

   },

   #information about target server

   'target_server' => {

      'hostname' => '', # used to see if we are on target

      'poolname' => 'backup', # name of the ZFS pool to import

      # if this is set, the dataset uses GELI, so we must decrypt and

      # mount it first

      'geli' => {

         'secureKey ' => {

            'label' => 'replica', # the GPT label of the key disk

            'fstype' => 'ufs', # filesystem type of the key disk

            'check_interval' => 15,

            'wait_timeout' => 300,

            'keyfile' => 'geli.key', # the name of the key file on the secureKey disk

         },

         'localKey' => 'e98c660cccdae1226550484d62caa2b72f60632ae0c607528aba1ac9e7bfbc9c', # hex representation of the local key part

         'target' => '/media/geli.key', # location to create the combined keyfile

         'poolname' => 'backup', # name of the ZFS pool to import

         'diskList' => [ 

            'da0',

            'da1'

            ], # list of disks to try to mount the dataset from

      },

      'report' => {

         'email' => '',

         'subject' => '',

         'targetDrive' => {

            'fstype' => 'msdos', # filesystem type of the report drive

            'label' => 'sneakernet',

            'mount_point' => '',

         }

      }

   },

   'transport' => {

      # this is the GPT label of the sneakernet disk

      'disk_label' => 'sneakernet',

      # this is the file system type. Not needed if ufs

      'fstype' => 'ufs',

      # where we want to mount it

      'mount_point' => '/mnt/sneakernet',

      # amount of time to wait for the disk to appear

      'timeout' => 600,

      # How often to check for the disk (seconds), message displayed every interval

      'check_interval' => 15,

      # if set, all files will be encrypted with this key/IV during transport

      'encryption' => {

         'key'    => '', # openssl rand 32 | xxd -p | tr -d '\n' > test.key

         'IV'     => '00000000000000000000000000000000',

      },

   },

   'datasets' => {

      'dataset1' => {

         'source' => 'pool/dataset1',

         'target' => 'backup/dataset1',

         'filename' => 'dataset1'

      },

      'files_share'  => {

         'source' => 'pool/files_share',

         'target' => 'backup/files_share',

         'filename' => 'files_share'

      },

   }

};

# read the status file and return as list. If the file doesn't exits, returns an empty list

sub getStatusFile {

   my $filename = shift;

   # read in history/status file

   my @lines = ();

   if ( -e $filename && open my $fh, '<', $filename ) {

      chomp( @lines = <$fh> );

      close $fh;

      logMsg("Read status file '$filename' with contents:\n" . join( "\n", @lines ) . "\n");

   } else {

      logMsg("Error: could not read status file '$filename', assuming a fresh start: $!");

   }

   return \@lines;

}

# write the status list to file

sub writeStatusFile {

   my ( $filename, $statusList ) = @_;

   # backup existing status file

   if ( -e $filename ) {

      rename( $filename, "$filename.bak" ) or do {

         logMsg("Error: could not backup existing status file '$filename': $!");

         die;

      };

   }

   # write new status file

   if ( open my $fh, '>', $filename ) {

      foreach my $line ( @$statusList ) {

         print $fh "$line\n";

      }

      close $fh;

      logMsg("Wrote status file '$filename' with contents:\n" . join( "\n", @$statusList ) . "\n");

   } else {

      logMsg("Error: could not write status file '$filename': $!");

      die;

   }

}

# simple sub to take root/dataset/datset/dataset and turn it into

# dataset.dataset.dataset

sub replaceSlashWithDot {

   my $string = shift;

   my @parts = split( "/", $string );

   shift @parts;

   return join( '.', @parts );

}

# perform replication on source server

# $config - configuration hashref

# $statusList - list of last snapshots replicated for each dataset in previous replications

# return new status list after replication containing updated last snapshots

# this script will actually replicate the datasets to the sneakernet disk

sub doSourceReplication {

   my ($config, $statusList) = @_;

   my $newStatus = [];

   foreach my $dataset ( sort keys %{$config->{datasets}} ) {

      logMsg("Processing dataset '$dataset'");

      # get list of all snapshots on dataset

      my $sourceList = [ runCmd( "zfs list -rt snap -H -o name $config->{datasets}->{$dataset}->{source} " ) ];

      # process dataset here

      my $commands = makeReplicateCommands($sourceList, $statusList, $newStatus );

      if ( %$commands ) {

         foreach my $cmd ( keys %$commands ) {

            my $command = $commands->{$cmd};

            $command .= " | openssl enc -aes-256-cbc -K $config->{transport}->{encryption}->{key} -iv $config->{transport}->{encryption}->{IV} " if $config->{transport}->{encryption}->{key};

            $command .= " > $config->{transport}->{mount_point}/" . replaceSlashWithDot($cmd);

            logMsg("Running command: $command");

            runCmd(  $command  ) unless $DEBUG;

         }

      } else {

         logMsg( "Nothing to do for $dataset" ); 

      }

   }

   return $newStatus;

}

# clean all files from a directory, but not any subdirectories

sub cleanDirectory {

   my $dirname = shift;

   logMsg( "Cleaning up $dirname of all files" );

   # clean up a directory

   opendir( my $dh, $dirname ) || fatalError( "Can not open $dirname: #!" );

   # get all file names, but leave directories alone

   my @files = map{ $dirname . "/$_" } grep { -f "$dirname/$_" } readdir($dh);

   closedir $dh;

   foreach my $file (@files) {

      unlink $file or warn "Could not unlink $file: #!\n";

   }

 }

# how to handle a fatal error

sub fatalError {

   my $message = shift;

   logMsg( $message );

   die;

}

##################### main program starts here #####################

# Example to create a random key for encryption/decryption:

# generate a random key with

# openssl rand 32 | xxd -p | tr -d '\n' > test.key

# If a YAML config file exists next to the script, load and merge it

$config = loadConfig($configFileName, $config );

# set some defaults

$config->{'status_file'} //= "$scriptFullPath.status";

# set log file name for sub logMsg in ZFS_Utils, and remove the old log if it exists

# Log file is only valid for one run

$logFileName = $config->{'log_file'} //= "$scriptFullPath.log";

# log only for one run

unlink ( $logFileName ) if -f $logFileName;

fatalError( "Invalid config file: missing source and/or target server" )

    unless (defined $config->{source_server} && defined $config->{target_server});

# mount the transport drive, fatal error if we can not find it

fatalError( "Unable to mount tranport drive with label $config->{transport}->{disk_label}" )

   unless $config->{transport}->{mount_point} =  mountDriveByLabel( $config->{transport} );

my $servername = `hostname -s`;

chomp $servername;

my $runningAs = $servername eq $config->{source_server}->{hostname} ? 'source' :

                $servername eq $config->{target_server}->{hostname} ? 'target' : 'unknown';

if ( $runningAs eq 'source' ) {

    logMsg "Running as source server";

    # remove all files from transport disk, but leave all subdirectories alone

    cleanDirectory( $config->{transport}->{mount_point} );

    my $statusList = getStatusFile($config->{status_file});

    $statusList = doSourceReplication($config, $statusList);

    writeStatusFile($config->{status_file}, $statusList);

} elsif ( $runningAs eq 'target' ) {

    logMsg "Running as target server";

    die "Target Server code not complete\n";

    die "GELI target server logic not yet implemented\n" if ( defined $config->{target_server}->{geli} );

    mountGeli( $config->{target_server}->{geli} ) if ( defined $config->{target_server}->{geli} );

} else {

    logMsg "This server ($servername) is neither source nor target server as per config\n";

    die;

}

# add disk space utilization information on transport to the log

logMsg( "Disk space utilization on transport disk:\n" . runCmd( "df -h $config->{transport}->{mount_point}" ) . "\n" );

# add information about the server (zpools) to the log

logMsg( "Zpools on server $servername:\n" . join( "\n", runCmd( "zpool list" ) ) . "\n" );

# unmount the sneakernet drive

`umount $config->{transport}->{mount_point}`;

# and remove the directory

rmdir $config->{transport}->{mount_point};

sendReport( $config->{$runningAs}->{report}, "sneakernet replication completed on server $servername", $config->{log_file} );

1;

#`cat $config->{input} | openssl enc -aes-256-cbc -K $config->{key} -iv $config->{IV} > $config->{output}`;

# this will decrypt $config->{output} to stdout

#`cat $config->{output} | openssl enc -aes-256-cbc -d -K $config->{key} -iv $config->{IV} > test.out`;