WebSVN – havirt – Diff – /trunk/cluster.pm

 # (new entry in config file). --dryrun will simply display the commands sent, and --nodryrun will execute them.
+#
 # v1.3.0 20250514 RWR
 # Modified so it will not issue error codes if we have done iterations but not perfectly in balance. Also, added message
 # 'already in balance' if it is as good as we can get it.
 #
+# v1.3.2 20260102 RWR
+# Refactored all command executions to use centralized execute() function
+# Updated updateISCITargets function to use execute() for all iscsiadm commands
 package cluster;
 use strict;
 # define the version number
 # see https://metacpan.org/pod/release/JPEACOCK/version-0.97/lib/version.pod
 use version;
-our $VERSION = version->declare("1.3.1");
+our $VERSION = version->declare("1.3.2");
 use Data::Dumper;
 use Exporter;
    push @return, "Processing iSCSI targets on $node";
    print Dumper( keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) if $main::config->{'flags'}->{'debug'};
    foreach my $server (keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) {
       print "\n" . '-'x40 . "\nGetting targets on server $server\n" . '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
       $command = &main::makeCommand( $node, "iscsiadm -m discovery -t st -p $server" );
-      my @list = `$command`;
+      my @list = &main::execute($command);
       chomp @list;
       # @list contains lines of type
       # 10.19.209.2:3260,1 iqn.2014-11.net.dailydata.castor:simon0
       # split them apart and add them to the hash
       foreach my $entry ( @list ) {
       } # foreach
    } # while
    print "\n" . '-'x40 . "\nGetting active sessions\n". '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
    # now, get active sessions so we can filter them
    $command = &main::makeCommand( $node, "iscsiadm -m session" );
-   my @activeSessions = `$command`;;
+   my @activeSessions = &main::execute($command);
    chomp @activeSessions;
    foreach my $session ( @activeSessions ) {
       $session =~ m/^.*[^0-9:.]([0-9,:.]+).*(iqn\S*)/;
       my ( $portal,$targetName ) = ( $1,$2 );
       print "$portal\t$targetName" if $main::config->{'flags'}->{'verbose'};
          push @return, "Adding $targetName";
          $command = &main::makeCommand( $node, "iscsiadm -m node --targetname '$targetName' --portal '$portal' --login" );
          if ( $main::config->{'flags'}->{'dryrun'} ) {
             push @return, $command;
          } else {
-          `$command`;
+          &main::execute($command);
+         }
+      }
    } else {
       push @return, "No new entries";
+   }
       $return->{'cluster'}->{'used_memory'} += $return->{'node'}->{$node}->{'used_memory'};
       $return->{'cluster'}->{'used_vcpu'} += $return->{'node'}->{$node}->{'used_vcpu'};
       $return->{'cluster'}->{'domain_count'} += $return->{'node'}->{$node}->{'count'};
+   }
    # calculate the deviation for each active node in the cluster
-   $return->{'cluster'}->{'average_memory'} = $return->{'cluster'}->{'used_memory'} / $return->{'cluster'}->{'memory'};
+   $return->{'cluster'}->{'average_memory'} = $return->{'cluster'}->{'used_memory'} / $return->{'cluster'}->{'memory'} if $return->{'cluster'}->{'memory'};
    # get the deviation for each node
    # variance in the cluster is simply the average of all deviations
    $return->{'cluster'}->{'variance'} = 0;
+   $return->{'cluster'}->{'activeNodes'} = 0;
    foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
       # deviation is the square of the difference between this node and the cluster overall
       $return->{'node'}->{$node}->{'deviation'} = (
          $return->{'node'}->{$node}->{'average_memory'} / $return->{'cluster'}->{'average_memory'}
          ) ** 2;
       # we'll divide by number of active nodes after the loop
       $return->{'cluster'}->{'variance'} += $return->{'node'}->{$node}->{'deviation'};
+      # do not count nodes in maintenance mode as active
+      $return->{'cluster'}->{'activeNodes'}++ unless $main::statusDB->{'node'}->{$node}->{'maintenance'};
+   }
+   die "No active nodes in cluster\n" unless $return->{'cluster'}->{'activeNodes'};
-   $return->{'cluster'}->{'variance'} /= $return->{'cluster'}->{'count'};
+   # die "$return->{'cluster'}->{'variance'}\t$return->{'cluster'}->{'activeNodes'}\n";
+   $return->{'cluster'}->{'variance'} /= $return->{'cluster'}->{'activeNodes'} if $return->{'cluster'}->{'activeNodes'};
    # now, determine how much memory needs to be added (plus) or removed (minus) for each node
    # memory_needed is calculated by taking the total amount of memory and multiplying it by the cluster average memory
    # then subtracting whatever is already used
    foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
       if ( $main::statusDB->{'node'}->{$node}->{'maintenance'} ) {
    # get the current cluster status
    my $cluster = &getClusterStats();
    #die Dumper( $cluster ) . "\n";
    # show user what it looks like at first
    print "=== Starting Status ===\n\n" . &showBalanceReport( $cluster) unless $main::config->{'flags'}->{'quiet'};
+   # First, ensure all domains are removed from nodes in maintenance mode
+   foreach my $node (keys %{$main::statusDB->{'node'}}) {
+      if ( $main::statusDB->{'node'}->{$node}->{'maintenance'} ) {
+         my @domains = keys %{$main::statusDB->{'nodePopulation'}->{$node}->{'running'}};
+         if ( @domains ) {
+            print "Node $node is in maintenance mode with " . scalar(@domains) . " domains, evacuating...\n"
+               if $main::config->{'flags'}->{'verbose'};
+            foreach my $domain (@domains) {
+               # Skip domains that are also in maintenance
+               next if $main::statusDB->{'virt'}->{$domain}->{'maintenance'};
+               # Find a suitable target node (not in maintenance, with enough resources)
+               my $target = '';
+               foreach my $candidate (keys %{$main::statusDB->{'node'}}) {
+                  next if $main::statusDB->{'node'}->{$candidate}->{'maintenance'};
+                  next if $candidate eq $node;
+                  # Check if target has enough resources
+                  my $available = &main::getAvailableResources($candidate);
+                  if ($available->{'memory'} >= $main::statusDB->{'virt'}->{$domain}->{'memory'}) {
+                     $target = $candidate;
+                     last;
+                  }
+               }
+               if ($target) {
+                  $return .= &main::migrate($domain, $target, $node);
+                  delete $main::statusDB->{'nodePopulation'}->{$node}->{'running'}->{$domain};
+                  $main::statusDB->{'nodePopulation'}->{$target}->{'running'}->{$domain} = time;
+               } else {
+                  print "Warning: Could not find suitable target for $domain from maintenance node $node\n";
+               }
+            }
+            &main::forceScan() unless $main::config->{'flags'}->{'dryrun'} || $main::config->{'flags'}->{'testing'};
+            # Refresh cluster stats after evacuation
+            $cluster = &getClusterStats();
+         }
+      }
+   }
    # we will do a loop to get the variance within our preferred range ($main::config->{ 'balance variance'})
    # however, we will only do a maximum number of iterations ($main::config->{ 'balance maxiterations'})
    my $iterations = defined $main::config->{ 'balance_max_iterations'} && $main::config->{ 'balance_max_iterations'} ? $main::config->{ 'balance_max_iterations'} : 10;
-   $main::config->{ 'balance_max_variance'} = 1.1 unless defined $main::config->{ 'balance_max_variance'};
+   $main::config->{ 'balance_max_variance'} = 0.5 unless defined $main::config->{ 'balance_max_variance'};
    # continue until our variance is where we want it, or we have tried too many times.
    while ( $iterations-- && $cluster->{'cluster'}->{'variance'} > $main::config->{ 'balance_max_variance'} ) {
       my $actions = &moveThings( $cluster );
       if ( my $output = &doActions( $actions ) ) {
          $return .= $output;
    my $transfer;
    my $from = '';
    my $to = '';
    # find smallest and largest "memory needed" in group. Note that if a node has too much, the number is negative and
    # for too little (ie, needs additional), the number is positive
+   # Skip nodes in maintenance mode for the target, but maintenance nodes can be source
    foreach my $node (keys %{$stats->{'node'} } ) {
       #print "Checking $node\n";
       if ( $from ) {
          $from = $node if $stats->{'node'}->{$from}->{'memory_needed'} > $stats->{'node'}->{$node}->{'memory_needed'};
+         # Only consider non-maintenance nodes as targets
+         if ( !$stats->{'node'}->{$node}->{'maintenance'} ) {
+            if ( $to ) {
-         $to = $node if $stats->{'node'}->{$to}->{'memory_needed'} < $stats->{'node'}->{$node}->{'memory_needed'};
+               $to = $node if $stats->{'node'}->{$to}->{'memory_needed'} < $stats->{'node'}->{$node}->{'memory_needed'};
+            } else {
+               $to = $node;
+            }
+         }
-      } else { # just initialize everything to this node
+      } else { # initialize $from to this node, $to only if not in maintenance
-         $from = $to = $node;
+         $from = $node;
+         $to = $node unless $stats->{'node'}->{$node}->{'maintenance'};
       } #if .. else
    } # foreach
+   # Don't move anything if no valid target nodes available
+   return $actions unless $to;
    # this is a poor mans min. we want to transfer the least number of bytes, ie what $from can spare, or what $to can accept
    # we need the smallest of what $from can give and $to can accept
    $transfer = abs( abs( $stats->{'node'}->{$from}->{'memory_needed'} ) > abs( $stats->{'node'}->{$to}->{'memory_needed'} ) ?
                $stats->{'node'}->{$to}->{'memory_needed'} : $stats->{'node'}->{$from}->{'memory_needed'} );
    # die "Transfer " . &humanReadable($transfer) ." bytes from $from to $to\n";

Subversion Repositories havirt

(root)/trunk/cluster.pm – Rev 46 → 47