Subversion Repositories zfs_utils

#! /usr/bin/env perl

use strict;
use warnings;

use FindBin;
use lib "$FindBin::Bin/..";
use Data::Dumper;
use ZFS_Utils qw(loadConfig shredFile logMsg makeReplicateCommands mountDriveByLabel mountGeli runCmd $logFileName $displayLogsOnConsole);

# if set, will not actually write files to disk
my $DEBUG = 0;

# set the log file to be next to this script
$logFileName = "$FindBin::Bin/sneakernet.log";
# log only for one run
unlink ( $logFileName ) if -f $logFileName;

# display all log messages on console in addition to the log file
$displayLogsOnConsole = 1;

my $configFileName = "$0.conf.yaml";

my $config = {
   # file created on source server to track last copyed dataset
   'status_file' => "$0.status",
   #information about source server
   'source_server' => {
      'hostname' => '', # used to see if we are on source
      'poolname' => '', # name of the ZFS pool to export
   },
   #information about target server
   'target_server' => {
      'hostname' => '', # used to see if we are on target
      'poolname' => '', # name of the ZFS pool to import
      # if this is set, the dataset uses GELI, so we must decrypt and
      # mount it first
      'geli' => {
         'keydiskname' => 'replica', # the GPT label of the key disk
         'keyfile' => 'geli.key', # the name of the key file on keydiskname
         'localKey' => 'e98c660cccdae1226550484d62caa2b72f60632ae0c607528aba1ac9e7bfbc9c', # hex representation of the local key part
         'target' => '/media/geli.key', # location to create the combined keyfile
         'poolname' => '', # name of the ZFS pool to import
         'diskList' => [ 
            '/dev/gpt/sneakernet_disk' 
            ], # list of disks to try to mount the dataset from
      }
   },
   
   'transport' => {
      # this is the GPT label of the sneakernet disk
      'disk_label' => 'sneakernet',
      # where we want to mount it
      'mount_point' => '/mnt/sneakernet',
      # amount of time to wait for the disk to appear
      'timeout' => 600,
      # if set, all files will be encrypted with this key/IV during transport
      'encryption' => {
         'key'    => '', # openssl rand 32 | xxd -p | tr -d '\n' > test.key
         'IV'     => '00000000000000000000000000000000',
      },
   },
   'datasets' => {
      'iscsi' => {
         'source' => 'storage/backup/iscsi',
         'target' => 'storage/backup/iscsi',
         'filename' => 'iscsi'
      },
      'files_share'  => {
         'source' => 'storage/backup/files_share',
         'target' => 'storage/backup/files_share',
         'filename' => 'files_share'
      },
   }
};

# read the status file and return as list
sub getStatusFile {
   my $filename = shift;
   # read in history/status file
   my @lines;
   if ( -e $filename && open my $fh, '<', $filename ) {
      chomp( @lines = <$fh> );
      close $fh;
      logMsg("Read status file '$filename' with contents:\n" . join( "\n", @lines ) . "\n");
   } else {
      logMsg("Error: could not read status file '$filename': $!");
      die;
   }
   return \@lines;
}

# write the status list to file
sub writeStatusFile {
   my ( $filename, $statusList ) = @_;
   # backup existing status file
   if ( -e $filename ) {
      rename( $filename, "$filename.bak" ) or do {
         logMsg("Error: could not backup existing status file '$filename': $!");
         die;
      };
   }
   # write new status file
   if ( open my $fh, '>', $filename ) {
      foreach my $line ( @$statusList ) {
         print $fh "$line\n";
      }
      close $fh;
      logMsg("Wrote status file '$filename' with contents:\n" . join( "\n", @$statusList ) . "\n");
   } else {
      logMsg("Error: could not write status file '$filename': $!");
      die;
   }
}

# simple sub to take root/dataset/datset/dataset and turn it into
# dataset.dataset.dataset
sub replaceSlashWithDot {
   my $string = shift;
   my @parts = split( "/", $string );
   shift @parts;
   return join( '.', @parts );
}

# perform replication on source server
# $config - configuration hashref
# $statusList - list of last snapshots replicated for each dataset in previous replications
# return new status list after replication containing updated last snapshots
# this script will actually replicate the datasets to the sneakernet disk
sub doSourceReplication {
   my ($config, $statusList) = @_;
   my $newStatus = [];
   foreach my $dataset ( sort keys %{$config->{datasets}} ) {
      logMsg("Processing dataset '$dataset'");
      # get list of all snapshots on dataset
      my $sourceList = [ runCmd( "zfs list -rt snap -H -o name $config->{datasets}->{$dataset}->{source} " ) ];
      # process dataset here
      my $commands = makeReplicateCommands($sourceList, $statusList, $newStatus );
      if ( %$commands ) {
         foreach my $cmd ( keys %$commands ) {
            my $command = $commands->{$cmd};
            $command .= " | openssl enc -aes-256-cbc -K $config->{transport}->{encryption}->{key} -iv $config->{transport}->{encryption}->{IV} " if $config->{transport}->{encryption}->{key};
            $command .= " > $config->{transport}->{mount_point}/" . replaceSlashWithDot($cmd);
            logMsg("Running command: $command");
            runCmd(  $command  ) unless $DEBUG;
         }
      } else {
         logMsg( "Nothing to do for $dataset" ); 
      }
   }
   return $newStatus;
}

# clean all files from a directory, but not any subdirectories
sub cleanDirectory {
   my $dirname = shift;
   logMsg( "Cleaning up $dirname of all files" );
   # clean up a directory
   opendir( my $dh, $dirname ) || fatalError( "Can not open $dirname: #!" );
   # get all file names, but leave directories alone
   my @files = map{ $dirname . "/$_" } grep { -f "$dirname/$_" } readdir($dh);
   closedir $dh;
   foreach my $file (@files) {
      unlink $file or warn "Could not unlink $file: #!\n";
   }
   return;
 }
    


# how to handle a fatal error
sub fatalError {
   my $message = shift;
   logMsg( $message );
   die;
}


##################### main program starts here #####################
# Example to create a random key for encryption/decryption:
# generate a random key with
# openssl rand 32 | xxd -p | tr -d '\n' > test.key

# If a YAML config file exists next to the script, load and merge it
$config = loadConfig($configFileName, $config );

# set some defaults
$config->{'status_file'} = "$0.status" unless ( defined $config->{'status_file'} );


fatalError( "Invalid config file: missing source and/or target server" )
    unless (defined $config->{source_server} && defined $config->{target_server});

# mount the transport drive, fatal error if we can not find it
fatalError( "Unable to mount tranport drive with label $config->{transport}->{disk_label}" )
   unless $config->{transport}->{mount_point} =  mountDriveByLabel( $config->{transport}->{disk_label}, $config->{transport}->{mount_point}, $config->{transport}->{timeout} );

my $servername = `hostname -s`;
chomp $servername;
if ( $servername eq $config->{source_server}->{hostname} ) {
    logMsg "Running as source server";
    # remove all files from transport disk, but leave all subdirectories alone
    cleanDirectory( $config->{transport}->{mount_point} );
    my $statusList = getStatusFile($config->{status_file});
    $statusList = doSourceReplication($config, $statusList);
    writeStatusFile($config->{status_file}, $statusList);
    # source server logic here
} elsif ( $servername eq $config->{target_server}->{hostname} ) {
    logMsg "Running as target server";
    die "Target Server code not complete\n";
    die "GELI target server logic not yet implemented\n" if ( defined $config->{target_server}->{geli} );
    mountGeli( $config->{target_server}->{geli} ) if ( defined $config->{target_server}->{geli} );
} else {
    logMsg "This server ($servername) is neither source nor target server as per config\n";
    die;
}

# unmount the sneakernet drive
`umount $config->{transport}->{mount_point}`;
# and remove the directory
rmdir $config->{transport}->{mount_point};

1;


#`cat $config->{input} | openssl enc -aes-256-cbc -K $config->{key} -iv $config->{IV} > $config->{output}`;

# this will decrypt $config->{output} to stdout
#`cat $config->{output} | openssl enc -aes-256-cbc -d -K $config->{key} -iv $config->{IV} > test.out`;