#! /usr/bin/env perl

# Copyright (c) 2016, Daily Data, Inc.
#
# Permission to use, copy, modify, and/or distribute this software for any purpose with 
# or without fee is hereby granted, provided that the above copyright notice and this 
# permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD 
# TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN
# NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 
# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER 
# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# Name: createSystemImage.pl
# Author: R. W. Rodolico for Daily Data, Inc (http://dailydata.net)
#
# Description:
# Script to manage multiple, timed images of file systems to allow rapid recovery of
# corrupted or lost files.
#
# Information:
# In response to the prevelance of ransomware, I created this simple backup script to
# keep as many versions of a file system as possible. It is local to the hard drive
# and SHOULD NOT be used as a backup system. In case of system crash/loss, it will
# obviously not be useful for recovery.
#
# The script relies heavily on rsync and file systems which allow hard links, providing
# an efficient way to store full directory tree replications in a very efficient manner
#
# Script was developed on Debian Linux (wheezy) with an ext4 file system. The subroutine
# makePackageSelection is specific to Debian based systems which use dpkg.
#
# Program Flow:
# 1. use rsync to copy files from @systemFiles to $systemFileDirectory
# 2. Create package list in $packageSelectionFile
# 3. use rsync to copy files from %otherServers to $otherServerDirectory
# 4. Check for free disk space in $target, deleting old archives until sufficient space exists
# 5. Create new directory as $target/date_time_stamp and populate with most recent backup using cp -al
# 6. use rsync to replicate $source to $target/date_time_stamp
#
# NOTES: 
# the variables source and target are used inconsistenly when viewed overall. 
# For example, when copying system files or other servers, the variables for "target"
# are generally in the $source directory. Thus, steps 1-3 may have parameters with
# "target" in them which point to a subdirectory of $source, because that is the 
# target of the action.
#
# For best results, and security, $target should be a mounted partition outside the normal
# access (smb, nfs), and is recommended to be 2x the size of the maximum $source space.
# If space available on $target is ever lower than space used on $source x $freeSpaceRequired,
# it is possible to destroy all previous versions.
#
# History:
# v0.1 - 20160619, RWR
#    initial build

use warnings;
use strict;


my $source = '/home';    # source of backups
my $target = '/backups'; # root of target for backups
my $systemFileDirectory = $source . '/systemFiles'; # target of system files backup
my $otherServerDirectory = $source . '/otherServers'; # target for other server backup
my %otherServers = ( # list of other servers to back up via rsync
                     'xen' => [  # URL/name of server
                        '/etc',  # list of directories to grab
                        '/opt' 
                     ] 
                  );
my $packageSelectionFile = $systemFileDirectory . '/packages.selection'; # name of package selection
my @systemFiles = ( '/etc', '/root', '/opt' ); # local directories to back up
my $freeSpaceRequired = 2; # we want 2x du of $source on $target

# prepopulate array with directories on target which match YYYY-MM-DD_HHMMSS
my @currentTargetDirectories = sort grep { /^\d{4}-\d{2}-\d{2}_\d{6}$/ } `ls $target`;
chomp @currentTargetDirectories;


# Name:       getSystemFiles
# Parameters: $title - name of section
# Returns:    $title, surrounded by two lines of 40 '=' sign
#
# simple helper sub to make a section title easier to read
sub makeReportSection {
   my $title = shift;
   return '='x40 . "\n$title\n" . '='x40 . "\n";
}

# Name:       getSystemFiles
# Parameters: $systemFileDirectory - Place to put system files
#             @systemFiles - array of directories to copy
# Returns:    Output of rsync
# WARNINGS:   adds/modifies data on $source
#
# Gets specified directories from current system and copies them into the source
# directory. This gets everything in one spot, so we can back them up
sub getSystemFiles {
   my ( $systemFileDirectory, @systemFiles ) = @_;
   `mkdir -p $systemFileDirectory` unless -d $systemFileDirectory;
   my $command = 'rsync -av ' . join( ' ', @systemFiles ) . ' ' . $systemFileDirectory;
   return `$command`;
}

# Name:       makePackageSelection
# Parameters: $target - Fully qualified name of file to store data in
# Returns:    nothing
# WARNINGS:   Assumes this is a debian system with dpkg installed
#
# simple helper sub grab packages currently installed on system. Stores them in
# $target
sub makePackageSelection {
   my $target = shift;
   `dpkg --get-selections > $target`;
}


# Name:       getOtherServer
# Parameters: $targetDir - where we want to copy to, generally in /home (what is called source otherwise)
#             $servers   - hash containing 'server'->['dir', 'dir']   
# Returns:    Report of actions taken (output of rsync)
# WARNINGS:   adds/modifies data on $source
#
# for each server defined in %$servers, will rsync the directories into $targetDir
# Assumes each target server defined in /root/.ssh/config and public key allows
# password free login. Any other mechanism to allow password free connection allowed.
sub getOtherServer {
   my ( $targetDir, $servers ) = @_;
   my $report;
   # do each server in turn.
   foreach my $server ( keys %$servers ) {
      # make target directory as $thisTarget/server_name
      my $thisTarget = $targetDir . '/' . $server;
      `mkdir -p $thisTarget` unless -d $thisTarget;
      # for each server directory, do an rsync
      foreach my $sourceDirectory (  @{ $$servers{$server} } ) {
         $report .= "Synchronizing $server:$sourceDirectory to $thisTarget/\n";
         $report .= `rsync -av $server:$sourceDirectory $thisTarget/`;
      }
   }
      
   return $report;
}

# Name:       cleanTargetSpace
# Parameters: $sourceDir - where we want to copy to
#             $targetDir - where we want to copy to
#             $freeSpace - Amount of free space required on target, as a multiple of space used by source
#             $directoriesOnTarget - Array of existing directories on target, sorted
# Returns:    Report of actions taken
# WARNINGS:   $directoriesOnTarget is modified, with deleted directories removed from it
#
# Checks targetDir for enough space to make backup. SourceDir is checked for size (using du) and targetDir
# is assumed to be a partition mounted, so df is used to see what is available
# Directories are removed, oldest first, until the amount of free space is great than
# source space used * $freeSpace
sub cleanTargetSpace {
   my ( $sourceDir, $targetDir, $freeSpace, $directoriesOnTarget ) = @_;
   my $report;
   my $spaceRequired = `du -Psk $sourceDir`;
   my $spaceAvailable = 0; # make sure to run loop at least once
   $spaceRequired =~ m/^(\d+)/;
   $spaceRequired = $1;
   $report = "Source has $spaceRequired k\n";
   $spaceRequired *= $freeSpace;
   $report = "We are reserving $spaceRequired k\n";
   while ( $spaceAvailable < $spaceRequired ) {
      $spaceAvailable = `df -k $targetDir`;
      foreach my $line ( split( "\n", $spaceAvailable ) ) {
         if ( $line =~ m/(\d+)\s+(\d+\%)\s+$targetDir$/ ) {
            $spaceAvailable = $1;
            last;
         } # if
      } # foreach
      if ( $spaceAvailable < $spaceRequired ) {
         die "Do not have enough space on $targetDir to make a backup; aborting\n" unless @$directoriesOnTarget;
         my $toRemove = shift @$directoriesOnTarget;
         $report .= "Removing Target Directory [$toRemove] to free up space\n";
         $toRemove = $targetDir . '/' . $toRemove;
         `rm -fR $toRemove`;
      }
   } # while 
   $report .= "Target has $spaceAvailable k\n";
   return $report;
}

# Name:       makeTargetDir
# Parameters: $targetDir - where we want to copy to
#             $directoriesOnTarget - Array of existing directories on target, sorted
# Returns:    Report of actions taken
# WARNINGS:   $directoriesOnTarget is modified
#
# Creates a new directory based on current date/time stamp, then adds that
# directory name to the end of @$directoriesOnTarget.
# if there was already a backup there, it is assumed the one just before 
# is the most recent, and it is copied to the new directory
# copying is done via hard links to take advantage of a nice feature of
# rsync command later
sub makeTargetDir {
   my ( $targetDir, $directoriesOnTarget ) = @_;
   my $report;
   # check if we have an previous backup, and if so, use it to populate
   my $source = @$directoriesOnTarget ? $$directoriesOnTarget[-1] : '';
   # create a new directory for this backup. Format is YYYY-MM-DD_HHMMSS
   my $target = `date +"%Y-%m-%d_%H%M%S"`; chomp $target;
   # save the new directory at the end of @$directoriesOnTarget
   push @$directoriesOnTarget, $target;
   # prepend $targetDir to all targets
   $target = $targetDir . '/' . $target;
   $source = $targetDir . '/' . $source if $source;
   $report .= "Creating new backup directory $target\n";
   `mkdir -p $target`;
   # if we have a source directory, use it to pre-populate   
   if ( $source ) {
      # The target directory is seeded with the most recent backup we have
      # using hard links (cp -al). We can do this since rsync will copy files
      # by first putting them in a temporary file, then unlinking the target file
      # and then renaming, so rsync will leave the original target file alone
      $report .= "Copying new backup from $source\n";
      #$report .= "cp -al $source/* $target/\n";
      `cp -al $source/* $target/`;
   } else {
      $report .= "There are no previous backups; this is the first. If that is not correct, this means we have a serious problem\n";
   }
   return $report;
}

# Name:       doBackup
# Parameters: $sourceDir - where we're copying from
#             $targetDir - Where we're copying to
# Returns:    Output of rsync command
#
# Simply do an rsync into the target directory. The target directory must exist.
# When rsync copies a file, it will copy as a temp file, then unlink the target if it already
# exists, then rename the temp file.
# Thus, since makeTargetDir copied the last backup via hard links, those links will remain
# for all unmodified files
sub doBackup {
   my ( $sourceDir, $targetDir ) = @_;
   my $report = '';
   $report = `rsync -a --exclude-from '/etc/rsync.exclude' --verbose --links --perms --recursive --checksum --delete-excluded --force --stats --delete $sourceDir $targetDir`;
   return $report;
}

my $report;

# get System Files, mainly anything outside of /home we also want (placed in /home/$systemFileDirectory)
$report .= &makeReportSection("Getting System Files") . &getSystemFiles( $systemFileDirectory, @systemFiles );
# get current package selection list, in case we need to rebuild
&makePackageSelection ( $packageSelectionFile );
# Get any other servers we may want to back up
$report .= &makeReportSection("Getting other servers") . &getOtherServer( $otherServerDirectory, \%otherServers );
# We now have all our files under /home, so check disk space on target, cleaning old versions if necessary
$report .= &makeReportSection( "Checking Free Space") . &cleanTargetSpace(  $source, $target, $freeSpaceRequired, \@currentTargetDirectories );
# build a new directory, and copy the files from the most recent to it
$report .= &makeReportSection( "Creating new directory to hold backup") . &makeTargetDir( $target, \@currentTargetDirectories );
# finally, make the actual backup
$report .= &makeReportSection( "Performing Backup") . &doBackup( "$source/*" , "$target/" . $currentTargetDirectories[-1] );
# report goes to root
print $report;

1;