Rev 39 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed
#!/usr/bin/env perl
# All functions related to maniplating/reporting on cluster
# part of havirt.
# Copyright 2024 Daily Data, Inc.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following
# conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the distribution.
# Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# v0.0.1 20240602 RWR
# Initial setup
#
# v1.2.0 20240826 RWR
# Added some code to migrate domains if node placed in maintenance mode
# Added a lot of 'verbose' print lines, and modified for new flag structure
#
package cluster;
use warnings;
use strict;
# define the version number
# see https://metacpan.org/pod/release/JPEACOCK/version-0.97/lib/version.pod
use version;
our $VERSION = version->declare("1.2.0");
use Data::Dumper;
use Exporter;
our @ISA = qw( Exporter );
our @EXPORT = qw(
&list
&iscsi
);
sub help {
my @return;
push @return, 'cluster status';
push @return, "\t[--format|-f screen|tsv] - displays some stats on cluster resources used";
push @return, 'cluster balance';
push @return, "\tBalances resources by moving domains between nodes";
push @return, 'cluster iscsi';
push @return, "\tdisplays list of all iSCSI targets 'known' by system";
push @return, 'cluster iscsi add ip-or-dns-name';
push @return, "\tAdds iscsi target to system";
push @return, 'cluster iscsi delete ip-or-dns-name';
push @return, "\tDelete iSCSI target processed by system. ip-or-dns-name MUST be exact";
push @return, 'cluster iscsi update [node ...]';
push @return, "\tPerforms an update to add new iSCSI targets on one or more nodes";
push @return, "\tScans all iSCSI targets, looking for new shares on each, then performs";
push @return, "\ta login, adding it to the node. DOES NOT delete old targets at this";
push @return, "\ttime. If no nodes passed in, will perform function on all nodes not";
push @return, "\tin maintenance mode";
return join( "\n", @return ) . "\n";
}
sub getClusterStats {
my $return = {};
foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
$return->{'nodes'}->{$node}->{'node_memory'} = $main::statusDB->{'node'}->{$node}->{'memory'};
$return->{'nodes'}->{$node}->{'node_vcpu'} = $main::statusDB->{'node'}->{$node}->{'cpu_count'};
$return->{'nodes'}->{$node}->{'node_maintenance'} = $main::statusDB->{'node'}->{$node}->{'maintenance'};
$return->{'nodes'}->{$node}->{'domain_memory'} = 0;
$return->{'nodes'}->{$node}->{'domain_vcpu'} = 0;
$return->{'nodes'}->{$node}->{'domain_count'} = 0;
foreach my $domain ( keys %{ $main::statusDB->{'nodePopulation'}->{$node}->{'running'} } ) {
$return->{'nodes'}->{$node}->{'domain_memory'} += $main::statusDB->{'virt'}->{$domain}->{'memory'};
$return->{'nodes'}->{$node}->{'domain_vcpu'} += $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
$return->{'nodes'}->{$node}->{'domain_count'}++;
$return->{'nodes'}->{$node}->{'domains'}->{$domain}->{'memory'} = $main::statusDB->{'virt'}->{$domain}->{'memory'};
$return->{'nodes'}->{$node}->{'domains'}->{$domain}->{'vcpu'} = $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
}
if ( ! $main::statusDB->{'node'}->{$node}->{'maintenance'} ) { # do not include node resources if maintenance set
$return->{'total_memory'} += $return->{'nodes'}->{$node}->{'node_memory'};
$return->{'total_vcpu'} += $return->{'nodes'}->{$node}->{'node_vcpu'};
}
$return->{'total_count'} += $return->{'nodes'}->{$node}->{'domain_count'};
$return->{'domain_memory'} += $return->{'nodes'}->{$node}->{'domain_memory'};
$return->{'domain_vcpu'} += $return->{'nodes'}->{$node}->{'domain_vcpu'};
}
return $return;
}
sub status {
my $return = '';
&main::readDB();
my @header = ('Node','Threads','Memory','Domains','vcpu','mem_used', 'Status' );
my @data;
my $usedmem = 0;
my $usedcpu = 0;
my $availmem = 0;
my $availcpu = 0;
my $totalDomains = 0;
my $maintenance = 0;
foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
my $memory = 0;
my $vcpus = 0;
my $count = 0;
foreach my $domain ( keys %{ $main::statusDB->{'nodePopulation'}->{$node}->{'running'} } ) {
$memory += $main::statusDB->{'virt'}->{$domain}->{'memory'};
$vcpus += $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
$count++;
}
push @data, [ $node,$main::statusDB->{'node'}->{$node}->{cpu_count},$main::statusDB->{'node'}->{$node}->{memory},$count,$vcpus,$memory, $main::statusDB->{'node'}->{$node}->{maintenance} ? 'Maintenance' : 'Online' ];
$usedmem += $memory;
$usedcpu += $vcpus;
$totalDomains += $count;
$availmem += $main::statusDB->{'node'}->{$node}->{memory};
$availcpu += $main::statusDB->{'node'}->{$node}->{cpu_count};
$maintenance += $main::statusDB->{'node'}->{$node}->{maintenance} ? 0 : 1;
} # outer for
push @data, [ 'Total',$availcpu,$availmem,$totalDomains,$usedcpu,$usedmem, $maintenance ];
return &main::report( \@header, \@data );
}
# perform various functions on iSCSI target definitions
# on all nodes
sub iscsi {
my $action = shift;
my @return;
if ( $action && $action eq 'add' ) {
&main::readDB(1);
while ( my $target = shift ) {
$main::statusDB->{'cluster'}->{'iscsi'}->{$target} = '';
}
&main::writeDB();
} elsif ( $action && $action eq 'delete' ) {
my $target = shift;
&main::readDB(1);
delete $main::statusDB->{'cluster'}->{'iscsi'}->{$target} if exists $main::statusDB->{'cluster'}->{'iscsi'}->{$target};
&main::writeDB();
} elsif ( $action && $action eq 'update' ) {
&main::readDB();
# if they did not give us a node, do all of them
@_ = keys %{ $main::statusDB->{'node'} } unless @_;
while ( my $node = shift ) { # process each node on stack
if ( $main::statusDB->{'node'}->{$node}->{'maintenance'} ) {
print "Not processing node $node since it is in maintenance mode\n" if $main::config->{'flags'}->{'verbose'};
} else { # actually do the work
push @return, &updateISCITargets( $node );
}
} # while
}
&main::readDB();
push @return, "iSCSI targets are";
if ( $main::statusDB->{'cluster'}->{'iscsi'} ) {
push @return, join( "\n", keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } );
} else {
push @return, "None Defined";
}
return join( "\n", @return ) . "\n";
}
# updates iSCSI targets on $node
# scans each target defined and compares it to the current session
# adding new targets if they exist
# NOTE: does not delete targets which no longer exist on server
sub updateISCITargets {
my $node = shift;
my $command;
my %targets;
my @return;
push @return, "Processing iSCSI targets on $node";
print Dumper( keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) if $main::config->{'flags'}->{'debug'};
foreach my $server (keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) {
print "\n" . '-'x40 . "\nGetting targets on server $server\n" . '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
$command = &main::makeCommand( $node, "iscsiadm -m discovery -t st -p $server" );
my @list = `$command`;
chomp @list;
# @list contains lines of type
# 10.19.209.2:3260,1 iqn.2014-11.net.dailydata.castor:simon0
# split them apart and add them to the hash
foreach my $entry ( @list ) {
my ( $portal, $targetName ) = split( ' ', $entry );
# $portal has some extra info after a comma, so clean it up
$portal =~ m/^([0-9:.]+)/;
$portal = $1;
# some targets return multiple IP's for a given name, so
# only add them if they are in this IP
$targets{ $targetName } = $portal if $portal =~ m/^$server/;
print "$targetName\t$targets{ $targetName }\n" if $main::config->{'flags'}->{'verbose'};
} # foreach
} # while
print "\n" . '-'x40 . "\nGetting active sessions\n". '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
# now, get active sessions so we can filter them
$command = &main::makeCommand( $node, "iscsiadm -m session" );
my @activeSessions = `$command`;;
chomp @activeSessions;
foreach my $session ( @activeSessions ) {
$session =~ m/^.*[^0-9:.]([0-9,:.]+).*(iqn\S*)/;
my ( $portal,$targetName ) = ( $1,$2 );
print "$portal\t$targetName" if $main::config->{'flags'}->{'verbose'};
if ( exists( $targets{$targetName} ) ) {
print "\tNOT updating\n" if $main::config->{'flags'}->{'verbose'};
delete $targets{ $targetName };
} else {
print "Needs to be added\n" if $main::config->{'flags'}->{'verbose'};
}
}
# check if we have any new entries and bail if not
if ( scalar keys %targets ) {
# We have new entries, so run them;
foreach my $targetName ( sort keys %targets ) {
my $portal = $targets{$targetName};
push @return, "Adding $targetName";
$command = &main::makeCommand( $node, "iscsiadm -m node --targetname '$targetName' --portal '$portal' --login" );
if ( $main::config->{'flags'}->{'dryrun'} ) {
push @return, $command;
} else {
`$command`;
}
}
} else {
push @return, "No new entries";
}
return join( "\n", @return ) . "\n";
} # updateISCITargets
# Creates a balance report to show the user what went on
# $cluster is a hash created by sub getClusterStats, and possibly modified by
# the calling process
sub showBalanceReport {
my $cluster = shift;
my $variance = 0;
my $count = 0;
my @header = ('Node','Threads','Memory(G)','Domains','vcpu_alloc','mem_alloc(G)', 'vcpu%', 'mem%', 'Status', 'StdDev' );
my @data;
foreach my $node ( sort keys %{ $cluster->{'nodes'} } ) {
# get standard deviation
my $stddev = $cluster->{'nodes'}->{$node}->{'node_maintenance'} ? 0 :
(
( $cluster->{'nodes'}->{$node}->{'domain_memory'} / $cluster->{'nodes'}->{$node}->{'node_memory'} * 100 ) -
( $cluster->{'domain_memory'} / $cluster->{'total_memory'} * 100 )
) ** 2;
push @data, [
$node,
$cluster->{'nodes'}->{$node}->{'node_vcpu'},
sprintf( '%d', $cluster->{'nodes'}->{$node}->{'node_memory'}/1024/1024 ),
$cluster->{'nodes'}->{$node}->{'domain_count'},
$cluster->{'nodes'}->{$node}->{'domain_vcpu'},
$cluster->{'nodes'}->{$node}->{'domain_memory'}/1024/1024,
sprintf( '%2.0f%%', $cluster->{'nodes'}->{$node}->{'domain_vcpu'} / $cluster->{'nodes'}->{$node}->{'node_vcpu'} * 100 ),
sprintf( '%2.0f%%', $cluster->{'nodes'}->{$node}->{'domain_memory'} / $cluster->{'nodes'}->{$node}->{'node_memory'} * 100 ),
$cluster->{'nodes'}->{$node}->{'node_maintenance'} ? 'Maintenance' : '',
sprintf( "%d", $stddev )
];
$variance += $stddev;
$count++;
}
push @data, [
'All',
$cluster->{'total_vcpu'},
sprintf( '%d', $cluster->{'total_memory'}/1024/1024 ),
$cluster->{'total_count'},
$cluster->{'domain_vcpu'},
$cluster->{'domain_memory'}/1024/1024,
sprintf( '%2.0f%%', $cluster->{'domain_vcpu'} / $cluster->{'total_vcpu'} * 100 ),
sprintf( '%2.0f%%', $cluster->{'domain_memory'} / $cluster->{'total_memory'} * 100 ),
''
];
return &main::report( \@header, \@data ),sprintf( "%d", $variance / $count );
}
# attempt to balance the domains on the active (maintenance = false) nodes
# basically, we take what is currently working, and calculate the variance
# of it (see https://en.wikipedia.org/wiki/Standard_deviation). If that is
# over about a 10, we move things around, if possible, then check our variance
# again.
sub balance {
&main::readDB();
# get the current cluster status
my $cluster = &getClusterStats();
# for development, turn on verbose
$main::config->{'flags'}->{'verbose'} = 1;
# show user what it looks like at first
if ( $main::config->{'flags'}->{'verbose'} ) {
print "Starting Status\n\n";
my ($report, $variance) = &showBalanceReport( $cluster) ;
print $report;
print "Variance is $variance\n";
}
die;
#die Dumper( $cluster ) . "\n";
return "This function not implemented yet\n";
}