8 |
rodolico |
1 |
#!/usr/bin/env perl
|
|
|
2 |
|
|
|
3 |
# All functions related to maniplating/reporting on cluster
|
|
|
4 |
# part of havirt.
|
|
|
5 |
|
|
|
6 |
# Copyright 2024 Daily Data, Inc.
|
|
|
7 |
#
|
|
|
8 |
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following
|
|
|
9 |
# conditions are met:
|
|
|
10 |
#
|
|
|
11 |
# Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
|
12 |
# Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
|
|
|
13 |
# in the documentation and/or other materials provided with the distribution.
|
|
|
14 |
# Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived
|
|
|
15 |
# from this software without specific prior written permission.
|
|
|
16 |
#
|
|
|
17 |
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
|
|
|
18 |
# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
|
|
|
19 |
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
|
20 |
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
21 |
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
|
22 |
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
23 |
|
|
|
24 |
# v0.0.1 20240602 RWR
|
|
|
25 |
# Initial setup
|
26 |
rodolico |
26 |
#
|
|
|
27 |
# v1.2.0 20240826 RWR
|
|
|
28 |
# Added some code to migrate domains if node placed in maintenance mode
|
|
|
29 |
# Added a lot of 'verbose' print lines, and modified for new flag structure
|
|
|
30 |
#
|
8 |
rodolico |
31 |
|
26 |
rodolico |
32 |
|
8 |
rodolico |
33 |
package cluster;
|
|
|
34 |
|
|
|
35 |
use warnings;
|
|
|
36 |
use strict;
|
|
|
37 |
|
|
|
38 |
# define the version number
|
|
|
39 |
# see https://metacpan.org/pod/release/JPEACOCK/version-0.97/lib/version.pod
|
|
|
40 |
use version;
|
26 |
rodolico |
41 |
our $VERSION = version->declare("1.2.0");
|
8 |
rodolico |
42 |
|
|
|
43 |
|
|
|
44 |
use Data::Dumper;
|
|
|
45 |
|
|
|
46 |
use Exporter;
|
|
|
47 |
|
|
|
48 |
our @ISA = qw( Exporter );
|
|
|
49 |
our @EXPORT = qw(
|
|
|
50 |
&list
|
26 |
rodolico |
51 |
&iscsi
|
8 |
rodolico |
52 |
);
|
|
|
53 |
|
11 |
rodolico |
54 |
sub help {
|
|
|
55 |
my @return;
|
|
|
56 |
push @return, 'cluster status';
|
|
|
57 |
push @return, "\t[--format|-f screen|tsv] - displays some stats on cluster resources used";
|
39 |
rodolico |
58 |
push @return, 'cluster balance';
|
|
|
59 |
push @return, "\tBalances resources by moving domains between nodes";
|
26 |
rodolico |
60 |
push @return, 'cluster iscsi';
|
|
|
61 |
push @return, "\tdisplays list of all iSCSI targets 'known' by system";
|
|
|
62 |
push @return, 'cluster iscsi add ip-or-dns-name';
|
|
|
63 |
push @return, "\tAdds iscsi target to system";
|
|
|
64 |
push @return, 'cluster iscsi delete ip-or-dns-name';
|
|
|
65 |
push @return, "\tDelete iSCSI target processed by system. ip-or-dns-name MUST be exact";
|
|
|
66 |
push @return, 'cluster iscsi update [node ...]';
|
|
|
67 |
push @return, "\tPerforms an update to add new iSCSI targets on one or more nodes";
|
|
|
68 |
push @return, "\tScans all iSCSI targets, looking for new shares on each, then performs";
|
|
|
69 |
push @return, "\ta login, adding it to the node. DOES NOT delete old targets at this";
|
|
|
70 |
push @return, "\ttime. If no nodes passed in, will perform function on all nodes not";
|
|
|
71 |
push @return, "\tin maintenance mode";
|
|
|
72 |
|
11 |
rodolico |
73 |
return join( "\n", @return ) . "\n";
|
|
|
74 |
}
|
|
|
75 |
|
40 |
rodolico |
76 |
sub getClusterStats {
|
|
|
77 |
my $return = {};
|
|
|
78 |
foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
|
|
|
79 |
$return->{'nodes'}->{$node}->{'node_memory'} = $main::statusDB->{'node'}->{$node}->{'memory'};
|
|
|
80 |
$return->{'nodes'}->{$node}->{'node_vcpu'} = $main::statusDB->{'node'}->{$node}->{'cpu_count'};
|
|
|
81 |
$return->{'nodes'}->{$node}->{'node_maintenance'} = $main::statusDB->{'node'}->{$node}->{'maintenance'};
|
|
|
82 |
|
|
|
83 |
$return->{'nodes'}->{$node}->{'domain_memory'} = 0;
|
|
|
84 |
$return->{'nodes'}->{$node}->{'domain_vcpu'} = 0;
|
|
|
85 |
$return->{'nodes'}->{$node}->{'domain_count'} = 0;
|
|
|
86 |
foreach my $domain ( keys %{ $main::statusDB->{'nodePopulation'}->{$node}->{'running'} } ) {
|
|
|
87 |
$return->{'nodes'}->{$node}->{'domain_memory'} += $main::statusDB->{'virt'}->{$domain}->{'memory'};
|
|
|
88 |
$return->{'nodes'}->{$node}->{'domain_vcpu'} += $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
|
|
|
89 |
$return->{'nodes'}->{$node}->{'domain_count'}++;
|
|
|
90 |
$return->{'nodes'}->{$node}->{'domains'}->{$domain}->{'memory'} = $main::statusDB->{'virt'}->{$domain}->{'memory'};
|
|
|
91 |
$return->{'nodes'}->{$node}->{'domains'}->{$domain}->{'vcpu'} = $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
|
|
|
92 |
}
|
|
|
93 |
if ( ! $main::statusDB->{'node'}->{$node}->{'maintenance'} ) { # do not include node resources if maintenance set
|
|
|
94 |
$return->{'total_memory'} += $return->{'nodes'}->{$node}->{'node_memory'};
|
|
|
95 |
$return->{'total_vcpu'} += $return->{'nodes'}->{$node}->{'node_vcpu'};
|
|
|
96 |
}
|
|
|
97 |
$return->{'total_count'} += $return->{'nodes'}->{$node}->{'domain_count'};
|
|
|
98 |
$return->{'domain_memory'} += $return->{'nodes'}->{$node}->{'domain_memory'};
|
|
|
99 |
$return->{'domain_vcpu'} += $return->{'nodes'}->{$node}->{'domain_vcpu'};
|
|
|
100 |
|
|
|
101 |
}
|
|
|
102 |
return $return;
|
|
|
103 |
}
|
|
|
104 |
|
10 |
rodolico |
105 |
sub status {
|
|
|
106 |
my $return = '';
|
13 |
rodolico |
107 |
&main::readDB();
|
25 |
rodolico |
108 |
my @header = ('Node','Threads','Memory','Domains','vcpu','mem_used', 'Status' );
|
10 |
rodolico |
109 |
my @data;
|
|
|
110 |
my $usedmem = 0;
|
|
|
111 |
my $usedcpu = 0;
|
|
|
112 |
my $availmem = 0;
|
|
|
113 |
my $availcpu = 0;
|
|
|
114 |
my $totalDomains = 0;
|
25 |
rodolico |
115 |
my $maintenance = 0;
|
13 |
rodolico |
116 |
foreach my $node (sort keys %{ $main::statusDB->{'node'} } ) {
|
10 |
rodolico |
117 |
my $memory = 0;
|
|
|
118 |
my $vcpus = 0;
|
|
|
119 |
my $count = 0;
|
13 |
rodolico |
120 |
foreach my $domain ( keys %{ $main::statusDB->{'nodePopulation'}->{$node}->{'running'} } ) {
|
|
|
121 |
$memory += $main::statusDB->{'virt'}->{$domain}->{'memory'};
|
|
|
122 |
$vcpus += $main::statusDB->{'virt'}->{$domain}->{'vcpu'};
|
10 |
rodolico |
123 |
$count++;
|
|
|
124 |
}
|
25 |
rodolico |
125 |
push @data, [ $node,$main::statusDB->{'node'}->{$node}->{cpu_count},$main::statusDB->{'node'}->{$node}->{memory},$count,$vcpus,$memory, $main::statusDB->{'node'}->{$node}->{maintenance} ? 'Maintenance' : 'Online' ];
|
10 |
rodolico |
126 |
$usedmem += $memory;
|
|
|
127 |
$usedcpu += $vcpus;
|
|
|
128 |
$totalDomains += $count;
|
13 |
rodolico |
129 |
$availmem += $main::statusDB->{'node'}->{$node}->{memory};
|
|
|
130 |
$availcpu += $main::statusDB->{'node'}->{$node}->{cpu_count};
|
26 |
rodolico |
131 |
$maintenance += $main::statusDB->{'node'}->{$node}->{maintenance} ? 0 : 1;
|
10 |
rodolico |
132 |
} # outer for
|
25 |
rodolico |
133 |
push @data, [ 'Total',$availcpu,$availmem,$totalDomains,$usedcpu,$usedmem, $maintenance ];
|
10 |
rodolico |
134 |
return &main::report( \@header, \@data );
|
|
|
135 |
}
|
26 |
rodolico |
136 |
|
|
|
137 |
# perform various functions on iSCSI target definitions
|
|
|
138 |
# on all nodes
|
|
|
139 |
|
|
|
140 |
|
|
|
141 |
sub iscsi {
|
|
|
142 |
my $action = shift;
|
|
|
143 |
my @return;
|
|
|
144 |
if ( $action && $action eq 'add' ) {
|
|
|
145 |
&main::readDB(1);
|
|
|
146 |
while ( my $target = shift ) {
|
|
|
147 |
$main::statusDB->{'cluster'}->{'iscsi'}->{$target} = '';
|
|
|
148 |
}
|
|
|
149 |
&main::writeDB();
|
|
|
150 |
} elsif ( $action && $action eq 'delete' ) {
|
|
|
151 |
my $target = shift;
|
|
|
152 |
&main::readDB(1);
|
|
|
153 |
delete $main::statusDB->{'cluster'}->{'iscsi'}->{$target} if exists $main::statusDB->{'cluster'}->{'iscsi'}->{$target};
|
|
|
154 |
&main::writeDB();
|
|
|
155 |
} elsif ( $action && $action eq 'update' ) {
|
|
|
156 |
&main::readDB();
|
|
|
157 |
# if they did not give us a node, do all of them
|
|
|
158 |
@_ = keys %{ $main::statusDB->{'node'} } unless @_;
|
|
|
159 |
while ( my $node = shift ) { # process each node on stack
|
|
|
160 |
if ( $main::statusDB->{'node'}->{$node}->{'maintenance'} ) {
|
|
|
161 |
print "Not processing node $node since it is in maintenance mode\n" if $main::config->{'flags'}->{'verbose'};
|
|
|
162 |
} else { # actually do the work
|
|
|
163 |
push @return, &updateISCITargets( $node );
|
|
|
164 |
}
|
|
|
165 |
} # while
|
|
|
166 |
}
|
|
|
167 |
&main::readDB();
|
|
|
168 |
push @return, "iSCSI targets are";
|
|
|
169 |
if ( $main::statusDB->{'cluster'}->{'iscsi'} ) {
|
|
|
170 |
push @return, join( "\n", keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } );
|
|
|
171 |
} else {
|
|
|
172 |
push @return, "None Defined";
|
|
|
173 |
}
|
|
|
174 |
return join( "\n", @return ) . "\n";
|
|
|
175 |
}
|
|
|
176 |
|
|
|
177 |
# updates iSCSI targets on $node
|
|
|
178 |
# scans each target defined and compares it to the current session
|
|
|
179 |
# adding new targets if they exist
|
|
|
180 |
# NOTE: does not delete targets which no longer exist on server
|
|
|
181 |
sub updateISCITargets {
|
|
|
182 |
my $node = shift;
|
|
|
183 |
my $command;
|
|
|
184 |
my %targets;
|
|
|
185 |
my @return;
|
|
|
186 |
push @return, "Processing iSCSI targets on $node";
|
|
|
187 |
print Dumper( keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) if $main::config->{'flags'}->{'debug'};
|
|
|
188 |
foreach my $server (keys %{ $main::statusDB->{'cluster'}->{'iscsi'} } ) {
|
|
|
189 |
print "\n" . '-'x40 . "\nGetting targets on server $server\n" . '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
|
|
|
190 |
$command = &main::makeCommand( $node, "iscsiadm -m discovery -t st -p $server" );
|
|
|
191 |
my @list = `$command`;
|
|
|
192 |
chomp @list;
|
|
|
193 |
# @list contains lines of type
|
|
|
194 |
# 10.19.209.2:3260,1 iqn.2014-11.net.dailydata.castor:simon0
|
|
|
195 |
# split them apart and add them to the hash
|
|
|
196 |
foreach my $entry ( @list ) {
|
|
|
197 |
my ( $portal, $targetName ) = split( ' ', $entry );
|
|
|
198 |
# $portal has some extra info after a comma, so clean it up
|
|
|
199 |
$portal =~ m/^([0-9:.]+)/;
|
|
|
200 |
$portal = $1;
|
|
|
201 |
# some targets return multiple IP's for a given name, so
|
|
|
202 |
# only add them if they are in this IP
|
|
|
203 |
$targets{ $targetName } = $portal if $portal =~ m/^$server/;
|
|
|
204 |
print "$targetName\t$targets{ $targetName }\n" if $main::config->{'flags'}->{'verbose'};
|
|
|
205 |
} # foreach
|
|
|
206 |
} # while
|
|
|
207 |
print "\n" . '-'x40 . "\nGetting active sessions\n". '-'x40 . "\n" if $main::config->{'flags'}->{'verbose'};
|
|
|
208 |
# now, get active sessions so we can filter them
|
|
|
209 |
$command = &main::makeCommand( $node, "iscsiadm -m session" );
|
|
|
210 |
my @activeSessions = `$command`;;
|
|
|
211 |
chomp @activeSessions;
|
|
|
212 |
foreach my $session ( @activeSessions ) {
|
|
|
213 |
$session =~ m/^.*[^0-9:.]([0-9,:.]+).*(iqn\S*)/;
|
|
|
214 |
my ( $portal,$targetName ) = ( $1,$2 );
|
|
|
215 |
print "$portal\t$targetName" if $main::config->{'flags'}->{'verbose'};
|
|
|
216 |
if ( exists( $targets{$targetName} ) ) {
|
|
|
217 |
print "\tNOT updating\n" if $main::config->{'flags'}->{'verbose'};
|
|
|
218 |
delete $targets{ $targetName };
|
|
|
219 |
} else {
|
|
|
220 |
print "Needs to be added\n" if $main::config->{'flags'}->{'verbose'};
|
|
|
221 |
}
|
|
|
222 |
}
|
|
|
223 |
|
|
|
224 |
# check if we have any new entries and bail if not
|
|
|
225 |
if ( scalar keys %targets ) {
|
|
|
226 |
# We have new entries, so run them;
|
|
|
227 |
foreach my $targetName ( sort keys %targets ) {
|
|
|
228 |
my $portal = $targets{$targetName};
|
|
|
229 |
push @return, "Adding $targetName";
|
|
|
230 |
$command = &main::makeCommand( $node, "iscsiadm -m node --targetname '$targetName' --portal '$portal' --login" );
|
|
|
231 |
if ( $main::config->{'flags'}->{'dryrun'} ) {
|
|
|
232 |
push @return, $command;
|
|
|
233 |
} else {
|
|
|
234 |
`$command`;
|
|
|
235 |
}
|
|
|
236 |
}
|
|
|
237 |
} else {
|
|
|
238 |
push @return, "No new entries";
|
|
|
239 |
}
|
|
|
240 |
return join( "\n", @return ) . "\n";
|
39 |
rodolico |
241 |
} # updateISCITargets
|
|
|
242 |
|
40 |
rodolico |
243 |
# Creates a balance report to show the user what went on
|
|
|
244 |
# $cluster is a hash created by sub getClusterStats, and possibly modified by
|
|
|
245 |
# the calling process
|
|
|
246 |
sub showBalanceReport {
|
|
|
247 |
my $cluster = shift;
|
|
|
248 |
my $variance = 0;
|
|
|
249 |
my $count = 0;
|
|
|
250 |
my @header = ('Node','Threads','Memory(G)','Domains','vcpu_alloc','mem_alloc(G)', 'vcpu%', 'mem%', 'Status', 'StdDev' );
|
|
|
251 |
my @data;
|
|
|
252 |
foreach my $node ( sort keys %{ $cluster->{'nodes'} } ) {
|
|
|
253 |
# get standard deviation
|
|
|
254 |
my $stddev = $cluster->{'nodes'}->{$node}->{'node_maintenance'} ? 0 :
|
|
|
255 |
(
|
|
|
256 |
( $cluster->{'nodes'}->{$node}->{'domain_memory'} / $cluster->{'nodes'}->{$node}->{'node_memory'} * 100 ) -
|
|
|
257 |
( $cluster->{'domain_memory'} / $cluster->{'total_memory'} * 100 )
|
|
|
258 |
) ** 2;
|
|
|
259 |
|
|
|
260 |
push @data, [
|
|
|
261 |
$node,
|
|
|
262 |
$cluster->{'nodes'}->{$node}->{'node_vcpu'},
|
|
|
263 |
sprintf( '%d', $cluster->{'nodes'}->{$node}->{'node_memory'}/1024/1024 ),
|
|
|
264 |
$cluster->{'nodes'}->{$node}->{'domain_count'},
|
|
|
265 |
$cluster->{'nodes'}->{$node}->{'domain_vcpu'},
|
|
|
266 |
$cluster->{'nodes'}->{$node}->{'domain_memory'}/1024/1024,
|
|
|
267 |
sprintf( '%2.0f%%', $cluster->{'nodes'}->{$node}->{'domain_vcpu'} / $cluster->{'nodes'}->{$node}->{'node_vcpu'} * 100 ),
|
|
|
268 |
sprintf( '%2.0f%%', $cluster->{'nodes'}->{$node}->{'domain_memory'} / $cluster->{'nodes'}->{$node}->{'node_memory'} * 100 ),
|
|
|
269 |
$cluster->{'nodes'}->{$node}->{'node_maintenance'} ? 'Maintenance' : '',
|
|
|
270 |
sprintf( "%d", $stddev )
|
|
|
271 |
];
|
|
|
272 |
$variance += $stddev;
|
|
|
273 |
$count++;
|
|
|
274 |
}
|
|
|
275 |
push @data, [
|
|
|
276 |
'All',
|
|
|
277 |
$cluster->{'total_vcpu'},
|
|
|
278 |
sprintf( '%d', $cluster->{'total_memory'}/1024/1024 ),
|
|
|
279 |
$cluster->{'total_count'},
|
|
|
280 |
$cluster->{'domain_vcpu'},
|
|
|
281 |
$cluster->{'domain_memory'}/1024/1024,
|
|
|
282 |
sprintf( '%2.0f%%', $cluster->{'domain_vcpu'} / $cluster->{'total_vcpu'} * 100 ),
|
|
|
283 |
sprintf( '%2.0f%%', $cluster->{'domain_memory'} / $cluster->{'total_memory'} * 100 ),
|
|
|
284 |
''
|
|
|
285 |
];
|
|
|
286 |
return &main::report( \@header, \@data ),sprintf( "%d", $variance / $count );
|
|
|
287 |
}
|
|
|
288 |
|
|
|
289 |
# attempt to balance the domains on the active (maintenance = false) nodes
|
|
|
290 |
# basically, we take what is currently working, and calculate the variance
|
|
|
291 |
# of it (see https://en.wikipedia.org/wiki/Standard_deviation). If that is
|
|
|
292 |
# over about a 10, we move things around, if possible, then check our variance
|
|
|
293 |
# again.
|
39 |
rodolico |
294 |
sub balance {
|
40 |
rodolico |
295 |
&main::readDB();
|
|
|
296 |
# get the current cluster status
|
|
|
297 |
my $cluster = &getClusterStats();
|
|
|
298 |
# for development, turn on verbose
|
|
|
299 |
$main::config->{'flags'}->{'verbose'} = 1;
|
|
|
300 |
# show user what it looks like at first
|
|
|
301 |
if ( $main::config->{'flags'}->{'verbose'} ) {
|
|
|
302 |
print "Starting Status\n\n";
|
|
|
303 |
my ($report, $variance) = &showBalanceReport( $cluster) ;
|
|
|
304 |
print $report;
|
|
|
305 |
print "Variance is $variance\n";
|
|
|
306 |
}
|
|
|
307 |
|
|
|
308 |
die;
|
|
|
309 |
#die Dumper( $cluster ) . "\n";
|
39 |
rodolico |
310 |
return "This function not implemented yet\n";
|
|
|
311 |
}
|
40 |
rodolico |
312 |
|