Subversion Repositories zfs_utils

#! /usr/bin/env perl

use strict;

use warnings;

use FindBin;

use lib "$FindBin::Bin/..";

use Data::Dumper;

use ZFS_Utils qw(loadConfig shredFile logMsg makeReplicateCommands mountDriveByLabel mountGeli runCmd $logFileName $displayLogsOnConsole);

# if set, will not actually write files to disk

my $DEBUG = 0;

# set the log file to be next to this script

$logFileName = "$FindBin::Bin/sneakernet.log";

# log only for one run

unlink ( $logFileName ) if -f $logFileName;

# display all log messages on console in addition to the log file

$displayLogsOnConsole = 1;

my $configFileName = "$0.conf.yaml";

my $config = {

   # file created on source server to track last copyed dataset

   'status_file' => "$0.status",

   #information about source server

   'source_server' => {

      'hostname' => '', # used to see if we are on source

      'poolname' => '', # name of the ZFS pool to export

   },

   #information about target server

   'target_server' => {

      'hostname' => '', # used to see if we are on target

      'poolname' => '', # name of the ZFS pool to import

      # if this is set, the dataset uses GELI, so we must decrypt and

      # mount it first

      'geli' => {

         'keydiskname' => 'replica', # the GPT label of the key disk

         'keyfile' => 'geli.key', # the name of the key file on keydiskname

         'localKey' => 'e98c660cccdae1226550484d62caa2b72f60632ae0c607528aba1ac9e7bfbc9c', # hex representation of the local key part

         'target' => '/media/geli.key', # location to create the combined keyfile

         'poolname' => '', # name of the ZFS pool to import

         'diskList' => [ 

            '/dev/gpt/sneakernet_disk' 

            ], # list of disks to try to mount the dataset from

      }

   },

   'transport' => {

      # this is the GPT label of the sneakernet disk

      'disk_label' => 'sneakernet',

      # where we want to mount it

      'mount_point' => '/mnt/sneakernet',

      # amount of time to wait for the disk to appear

      'timeout' => 600,

      # if set, all files will be encrypted with this key/IV during transport

      'encryption' => {

         'key'    => '', # openssl rand 32 | xxd -p | tr -d '\n' > test.key

         'IV'     => '00000000000000000000000000000000',

      },

   },

   'datasets' => {

      'iscsi' => {

         'source' => 'storage/backup/iscsi',

         'target' => 'storage/backup/iscsi',

         'filename' => 'iscsi'

      },

      'files_share'  => {

         'source' => 'storage/backup/files_share',

         'target' => 'storage/backup/files_share',

         'filename' => 'files_share'

      },

   }

};

# read the status file and return as list

sub getStatusFile {

   my $filename = shift;

   # read in history/status file

   my @lines;

   if ( -e $filename && open my $fh, '<', $filename ) {

      chomp( @lines = <$fh> );

      close $fh;

      logMsg("Read status file '$filename' with contents:\n" . join( "\n", @lines ) . "\n");

   } else {

      logMsg("Error: could not read status file '$filename': $!");

      die;

   }

   return \@lines;

}

# write the status list to file

sub writeStatusFile {

   my ( $filename, $statusList ) = @_;

   # backup existing status file

   if ( -e $filename ) {

      rename( $filename, "$filename.bak" ) or do {

         logMsg("Error: could not backup existing status file '$filename': $!");

         die;

      };

   }

   # write new status file

   if ( open my $fh, '>', $filename ) {

      foreach my $line ( @$statusList ) {

         print $fh "$line\n";

      }

      close $fh;

      logMsg("Wrote status file '$filename' with contents:\n" . join( "\n", @$statusList ) . "\n");

   } else {

      logMsg("Error: could not write status file '$filename': $!");

      die;

   }

}

# simple sub to take root/dataset/datset/dataset and turn it into

# dataset.dataset.dataset

sub replaceSlashWithDot {

   my $string = shift;

   my @parts = split( "/", $string );

   shift @parts;

   return join( '.', @parts );

}

# perform replication on source server

# $config - configuration hashref

# $statusList - list of last snapshots replicated for each dataset in previous replications

# return new status list after replication containing updated last snapshots

# this script will actually replicate the datasets to the sneakernet disk

sub doSourceReplication {

   my ($config, $statusList) = @_;

   my $newStatus = [];

   foreach my $dataset ( sort keys %{$config->{datasets}} ) {

      logMsg("Processing dataset '$dataset'");

      # get list of all snapshots on dataset

      my $sourceList = [ runCmd( "zfs list -rt snap -H -o name $config->{datasets}->{$dataset}->{source} " ) ];

      # process dataset here

      my $commands = makeReplicateCommands($sourceList, $statusList, $newStatus );

      if ( %$commands ) {

         foreach my $cmd ( keys %$commands ) {

            my $command = $commands->{$cmd};

            $command .= " | openssl enc -aes-256-cbc -K $config->{transport}->{encryption}->{key} -iv $config->{transport}->{encryption}->{IV} " if $config->{transport}->{encryption}->{key};

            $command .= " > $config->{transport}->{mount_point}/" . replaceSlashWithDot($cmd);

            logMsg("Running command: $command");

            runCmd(  $command  ) unless $DEBUG;

         }

      } else {

         logMsg( "Nothing to do for $dataset" ); 

      }

   }

   return $newStatus;

}

# clean all files from a directory, but not any subdirectories

sub cleanDirectory {

   my $dirname = shift;

   logMsg( "Cleaning up $dirname of all files" );

   # clean up a directory

   opendir( my $dh, $dirname ) || fatalError( "Can not open $dirname: #!" );

   # get all file names, but leave directories alone

   my @files = map{ $dirname . "/$_" } grep { -f "$dirname/$_" } readdir($dh);

   closedir $dh;

   foreach my $file (@files) {

      unlink $file or warn "Could not unlink $file: #!\n";

   }

   return;

 }

# how to handle a fatal error

sub fatalError {

   my $message = shift;

   logMsg( $message );

   die;

}

##################### main program starts here #####################

# Example to create a random key for encryption/decryption:

# generate a random key with

# openssl rand 32 | xxd -p | tr -d '\n' > test.key

# If a YAML config file exists next to the script, load and merge it

$config = loadConfig($configFileName, $config );

# set some defaults

$config->{'status_file'} = "$0.status" unless ( defined $config->{'status_file'} );

fatalError( "Invalid config file: missing source and/or target server" )

    unless (defined $config->{source_server} && defined $config->{target_server});

# mount the transport drive, fatal error if we can not find it

fatalError( "Unable to mount tranport drive with label $config->{transport}->{disk_label}" )

   unless $config->{transport}->{mount_point} =  mountDriveByLabel( $config->{transport}->{disk_label}, $config->{transport}->{mount_point}, $config->{transport}->{timeout} );

my $servername = `hostname -s`;

chomp $servername;

if ( $servername eq $config->{source_server}->{hostname} ) {

    logMsg "Running as source server";

    # remove all files from transport disk, but leave all subdirectories alone

    cleanDirectory( $config->{transport}->{mount_point} );

    my $statusList = getStatusFile($config->{status_file});

    $statusList = doSourceReplication($config, $statusList);

    writeStatusFile($config->{status_file}, $statusList);

    # source server logic here

} elsif ( $servername eq $config->{target_server}->{hostname} ) {

    logMsg "Running as target server";

    die "Target Server code not complete\n";

    die "GELI target server logic not yet implemented\n" if ( defined $config->{target_server}->{geli} );

    mountGeli( $config->{target_server}->{geli} ) if ( defined $config->{target_server}->{geli} );

} else {

    logMsg "This server ($servername) is neither source nor target server as per config\n";

    die;

}

# unmount the sneakernet drive

`umount $config->{transport}->{mount_point}`;

# and remove the directory

rmdir $config->{transport}->{mount_point};

1;

#`cat $config->{input} | openssl enc -aes-256-cbc -K $config->{key} -iv $config->{IV} > $config->{output}`;

# this will decrypt $config->{output} to stdout

#`cat $config->{output} | openssl enc -aes-256-cbc -d -K $config->{key} -iv $config->{IV} > test.out`;