Subversion Repositories havirt

Rev

Rev 39 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
8 rodolico 1
#!/usr/bin/env perl
2
 
3
# All functions related to maniplating/reporting on cluster
4
# part of havirt.
5
 
6
# Copyright 2024 Daily Data, Inc.
7
# 
8
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following 
9
# conditions are met:
10
#
11
#   Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
12
#   Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 
13
#   in the documentation and/or other materials provided with the distribution.
14
#   Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived
15
#   from this software without specific prior written permission.
16
# 
17
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
18
# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
19
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
22
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23
 
24
# v0.0.1 20240602 RWR
25
# Initial setup
26 rodolico 26
#
27
# v1.2.0 20240826 RWR
28
# Added some code to migrate domains if node placed in maintenance mode
29
# Added a lot of 'verbose' print lines, and modified for new flag structure
30
#
8 rodolico 31
 
26 rodolico 32
 
8 rodolico 33
package cluster;
34
 
35
use warnings;
36
use strict;  
37
 
38
# define the version number
39
# see https://metacpan.org/pod/release/JPEACOCK/version-0.97/lib/version.pod
40
use version;
26 rodolico 41
our $VERSION = version->declare("1.2.0");
8 rodolico 42
 
43
 
44
use Data::Dumper;
45
 
46
use Exporter;
47
 
48
our @ISA = qw( Exporter );
49
our @EXPORT = qw( 
50
                  &list
26 rodolico 51
                  &iscsi
8 rodolico 52
                );
53
 
11 rodolico 54
sub help {
55
   my @return;
56
   push @return, 'cluster status';
57
   push @return, "\t[--format|-f screen|tsv] - displays some stats on cluster resources used";
39 rodolico 58
   push @return, 'cluster balance';
59
   push @return, "\tBalances resources by moving domains between nodes";
26 rodolico 60
   push @return, 'cluster iscsi';
61
   push @return, "\tdisplays list of all iSCSI targets 'known' by system";
62
   push @return, 'cluster iscsi add ip-or-dns-name';
63
   push @return, "\tAdds iscsi target to system";
64
   push @return, 'cluster iscsi delete  ip-or-dns-name';
65
   push @return, "\tDelete iSCSI target processed by system. ip-or-dns-name MUST be exact";
66
   push @return, 'cluster iscsi update [node ...]';
67
   push @return, "\tPerforms an update to add new iSCSI targets on one or more nodes";
68
   push @return, "\tScans all iSCSI targets, looking for new shares on each, then performs";
69
   push @return, "\ta login, adding it to the node. DOES NOT delete old targets at this";
70
   push @return, "\ttime. If no nodes passed in, will perform function on all nodes not";
71
   push @return, "\tin maintenance mode";
72
 
11 rodolico 73
   return join( "\n", @return ) . "\n";
74
}
75
 
40 rodolico 76
sub getClusterStats {
77
   my $return = {};
78
   foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
79
      $return->{'nodes'}->{$node}->{'node_memory'} = $main::statusDB->{'node'}->{$node}->{'memory'};
80
      $return->{'nodes'}->{$node}->{'node_vcpu'} = $main::statusDB->{'node'}->{$node}->{'cpu_count'};
81
      $return->{'nodes'}->{$node}->{'node_maintenance'} = $main::statusDB->{'node'}->{$node}->{'maintenance'};
82
 
83
      $return->{'nodes'}->{$node}->{'domain_memory'} = 0;
84
      $return->{'nodes'}->{$node}->{'domain_vcpu'} = 0;
85
      $return->{'nodes'}->{$node}->{'domain_count'} = 0;
86
      foreach my $domain ( keys %{ $main::statusDB->{'nodePopulation'}->{$node}->{'running'} } ) {
87
         $return->{'nodes'}->{$node}->{'domain_memory'} += $main::statusDB->{'virt'}->{$domain}->{'memory'};
88
         $return->{'nodes'}->{$node}->{'domain_vcpu'} += $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
89
         $return->{'nodes'}->{$node}->{'domain_count'}++;
90
         $return->{'nodes'}->{$node}->{'domains'}->{$domain}->{'memory'} = $main::statusDB->{'virt'}->{$domain}->{'memory'};
91
         $return->{'nodes'}->{$node}->{'domains'}->{$domain}->{'vcpu'} = $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
92
      }
93
      if ( ! $main::statusDB->{'node'}->{$node}->{'maintenance'} ) { # do not include node resources if maintenance set
94
         $return->{'total_memory'} += $return->{'nodes'}->{$node}->{'node_memory'};
95
         $return->{'total_vcpu'} += $return->{'nodes'}->{$node}->{'node_vcpu'};
96
      }
97
      $return->{'total_count'} += $return->{'nodes'}->{$node}->{'domain_count'};
98
      $return->{'domain_memory'} += $return->{'nodes'}->{$node}->{'domain_memory'};
99
      $return->{'domain_vcpu'} += $return->{'nodes'}->{$node}->{'domain_vcpu'};
100
 
101
   }
102
   return $return;
103
}
104
 
10 rodolico 105
sub status {
106
   my $return = '';
13 rodolico 107
   &main::readDB();
25 rodolico 108
   my @header = ('Node','Threads','Memory','Domains','vcpu','mem_used', 'Status' );
10 rodolico 109
   my @data;
110
   my $usedmem = 0;
111
   my $usedcpu = 0;
112
   my $availmem = 0;
113
   my $availcpu = 0;
114
   my $totalDomains = 0;
25 rodolico 115
   my $maintenance = 0;
13 rodolico 116
   foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
10 rodolico 117
      my $memory = 0;
118
      my $vcpus = 0;
119
      my $count = 0;
13 rodolico 120
      foreach my $domain ( keys %{ $main::statusDB->{'nodePopulation'}->{$node}->{'running'} } ) {
121
         $memory += $main::statusDB->{'virt'}->{$domain}->{'memory'};
122
         $vcpus += $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
10 rodolico 123
         $count++;
124
      }
25 rodolico 125
      push @data, [ $node,$main::statusDB->{'node'}->{$node}->{cpu_count},$main::statusDB->{'node'}->{$node}->{memory},$count,$vcpus,$memory, $main::statusDB->{'node'}->{$node}->{maintenance} ? 'Maintenance' : 'Online' ];
10 rodolico 126
      $usedmem += $memory;
127
      $usedcpu += $vcpus;
128
      $totalDomains += $count;
13 rodolico 129
      $availmem += $main::statusDB->{'node'}->{$node}->{memory};
130
      $availcpu += $main::statusDB->{'node'}->{$node}->{cpu_count};
26 rodolico 131
      $maintenance += $main::statusDB->{'node'}->{$node}->{maintenance} ? 0 : 1;
10 rodolico 132
   } # outer for
25 rodolico 133
   push @data, [ 'Total',$availcpu,$availmem,$totalDomains,$usedcpu,$usedmem, $maintenance ];
10 rodolico 134
   return &main::report( \@header, \@data );
135
}
26 rodolico 136
 
137
# perform various functions on iSCSI target definitions
138
# on all nodes
139
 
140
 
141
sub iscsi {
142
   my $action = shift;
143
   my @return;
144
   if ( $action && $action eq 'add' ) {
145
      &main::readDB(1);
146
      while ( my $target = shift ) {
147
         $main::statusDB->{'cluster'}->{'iscsi'}->{$target} = '';
148
      }
149
      &main::writeDB();
150
   } elsif ( $action && $action eq 'delete' ) {
151
      my $target = shift;
152
      &main::readDB(1);
153
      delete $main::statusDB->{'cluster'}->{'iscsi'}->{$target} if exists $main::statusDB->{'cluster'}->{'iscsi'}->{$target};
154
      &main::writeDB();
155
   } elsif ( $action && $action eq 'update' ) {
156
      &main::readDB();
157
      # if they did not give us a node, do all of them
158
      @_ = keys %{ $main::statusDB->{'node'} } unless @_;
159
      while ( my $node = shift ) { # process each node on stack
160
         if ( $main::statusDB->{'node'}->{$node}->{'maintenance'} ) {
161
            print "Not processing node $node since it is in maintenance mode\n" if $main::config->{'flags'}->{'verbose'};
162
         } else { # actually do the work
163
            push @return, &updateISCITargets( $node );
164
         }
165
      } # while
166
   }
167
   &main::readDB();
168
   push @return, "iSCSI targets are";
169
   if ( $main::statusDB->{'cluster'}->{'iscsi'} ) {
170
      push @return, join( "\n",  keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } );
171
   } else {
172
      push @return, "None Defined";
173
   }
174
   return join( "\n", @return ) . "\n";
175
}
176
 
177
# updates iSCSI targets on $node
178
# scans each target defined and compares it to the current session
179
# adding new targets if they exist
180
# NOTE: does not delete targets which no longer exist on server
181
sub updateISCITargets {
182
   my $node = shift;
183
   my $command;
184
   my %targets;
185
   my @return;
186
   push @return, "Processing iSCSI targets on $node";
187
   print Dumper( keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) if $main::config->{'flags'}->{'debug'};
188
   foreach my $server (keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) {
189
      print "\n" . '-'x40 . "\nGetting targets on server $server\n" . '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
190
      $command = &main::makeCommand( $node, "iscsiadm -m discovery -t st -p $server" );
191
      my @list = `$command`;
192
      chomp @list;
193
      # @list contains lines of type
194
      # 10.19.209.2:3260,1 iqn.2014-11.net.dailydata.castor:simon0
195
      # split them apart and add them to the hash
196
      foreach my $entry ( @list ) {
197
         my ( $portal, $targetName ) = split( ' ', $entry );
198
         # $portal has some extra info after a comma, so clean it up
199
         $portal =~ m/^([0-9:.]+)/;
200
         $portal = $1;
201
         # some targets return multiple IP's for a given name, so 
202
         # only add them if they are in this IP
203
         $targets{ $targetName } = $portal if $portal =~ m/^$server/;
204
         print "$targetName\t$targets{ $targetName }\n" if $main::config->{'flags'}->{'verbose'};
205
      } # foreach
206
   } # while
207
   print "\n" . '-'x40 . "\nGetting active sessions\n". '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
208
   # now, get active sessions so we can filter them
209
   $command = &main::makeCommand( $node, "iscsiadm -m session" );
210
   my @activeSessions = `$command`;;
211
   chomp @activeSessions;
212
   foreach my $session ( @activeSessions ) {
213
      $session =~ m/^.*[^0-9:.]([0-9,:.]+).*(iqn\S*)/;
214
      my ( $portal,$targetName ) = ( $1,$2 );
215
      print "$portal\t$targetName" if $main::config->{'flags'}->{'verbose'};
216
      if ( exists( $targets{$targetName} ) ) {
217
         print "\tNOT updating\n" if $main::config->{'flags'}->{'verbose'};
218
         delete $targets{ $targetName };
219
      } else {
220
         print "Needs to be added\n" if $main::config->{'flags'}->{'verbose'};
221
      }
222
   }
223
 
224
   # check if we have any new entries and bail if not
225
   if ( scalar keys %targets ) {
226
      # We have new entries, so run them;
227
      foreach my $targetName ( sort keys %targets ) {
228
         my $portal = $targets{$targetName};
229
         push @return, "Adding $targetName";
230
         $command = &main::makeCommand( $node, "iscsiadm -m node --targetname '$targetName' --portal '$portal' --login" );
231
         if ( $main::config->{'flags'}->{'dryrun'} ) {
232
            push @return, $command;
233
         } else {
234
          `$command`;
235
         }
236
      }
237
   } else {
238
      push @return, "No new entries";
239
   }
240
   return join( "\n", @return ) . "\n";
39 rodolico 241
} # updateISCITargets
242
 
40 rodolico 243
# Creates a balance report to show the user what went on
244
# $cluster is a hash created by sub getClusterStats, and possibly modified by
245
# the calling process
246
sub showBalanceReport {
247
   my $cluster = shift;
248
   my $variance = 0;
249
   my $count = 0;
250
   my @header = ('Node','Threads','Memory(G)','Domains','vcpu_alloc','mem_alloc(G)', 'vcpu%', 'mem%', 'Status', 'StdDev' );
251
   my @data;
252
   foreach my $node ( sort keys %{ $cluster->{'nodes'} } ) {
253
      # get standard deviation
254
      my $stddev = $cluster->{'nodes'}->{$node}->{'node_maintenance'} ? 0 : 
255
                   (
256
                      ( $cluster->{'nodes'}->{$node}->{'domain_memory'} / $cluster->{'nodes'}->{$node}->{'node_memory'} * 100 ) - 
257
                      ( $cluster->{'domain_memory'} / $cluster->{'total_memory'} * 100 )
258
                   ) ** 2;
259
 
260
      push @data, [
261
         $node, 
262
         $cluster->{'nodes'}->{$node}->{'node_vcpu'},
263
         sprintf( '%d', $cluster->{'nodes'}->{$node}->{'node_memory'}/1024/1024 ),
264
         $cluster->{'nodes'}->{$node}->{'domain_count'},
265
         $cluster->{'nodes'}->{$node}->{'domain_vcpu'},
266
         $cluster->{'nodes'}->{$node}->{'domain_memory'}/1024/1024,
267
         sprintf( '%2.0f%%', $cluster->{'nodes'}->{$node}->{'domain_vcpu'} / $cluster->{'nodes'}->{$node}->{'node_vcpu'} * 100 ),
268
         sprintf( '%2.0f%%', $cluster->{'nodes'}->{$node}->{'domain_memory'} / $cluster->{'nodes'}->{$node}->{'node_memory'} * 100 ),
269
         $cluster->{'nodes'}->{$node}->{'node_maintenance'} ? 'Maintenance' : '',
270
         sprintf( "%d", $stddev )
271
      ];
272
      $variance += $stddev;
273
      $count++;
274
   }
275
   push @data, [
276
         'All', 
277
         $cluster->{'total_vcpu'},
278
         sprintf( '%d', $cluster->{'total_memory'}/1024/1024 ),
279
         $cluster->{'total_count'},
280
         $cluster->{'domain_vcpu'},
281
         $cluster->{'domain_memory'}/1024/1024,
282
         sprintf( '%2.0f%%', $cluster->{'domain_vcpu'} / $cluster->{'total_vcpu'} * 100 ),
283
         sprintf( '%2.0f%%', $cluster->{'domain_memory'} / $cluster->{'total_memory'} * 100 ),
284
         ''
285
      ];
286
   return &main::report( \@header, \@data ),sprintf( "%d", $variance / $count );
287
}
288
 
289
# attempt to balance the domains on the active (maintenance = false) nodes
290
# basically, we take what is currently working, and calculate the variance
291
# of it (see https://en.wikipedia.org/wiki/Standard_deviation). If that is
292
# over about a 10, we move things around, if possible, then check our variance
293
# again.
39 rodolico 294
sub balance {
40 rodolico 295
   &main::readDB();
296
   # get the current cluster status
297
   my $cluster = &getClusterStats();
298
   # for development, turn on verbose
299
   $main::config->{'flags'}->{'verbose'} = 1;
300
   # show user what it looks like at first
301
   if ( $main::config->{'flags'}->{'verbose'} ) {
302
      print "Starting Status\n\n";
303
      my ($report, $variance) =  &showBalanceReport( $cluster) ;
304
      print $report;
305
      print "Variance is $variance\n";
306
   }
307
 
308
   die;
309
   #die Dumper( $cluster ) . "\n";
39 rodolico 310
   return "This function not implemented yet\n";
311
}
40 rodolico 312