BCV Management PERL Code

vbaperf:$ cat ./bcv_mgmt.pl.mod
#!/vbaperf/local/bin/perl

use Getopt::Std;
use Term::Cap;
use IO::Tee;
use POSIX;
use POSIX qw(setsid);
use POSIX qw(:errno_h :fcntl_h);
use File::Path;
use Archive::Tar::Streamed;
use Cwd;

# Turn on Auto buffering
$| = 1;

# Capture program directory
our $main_directory = getcwd;

# Verify that we are running as root
$get_id = POSIX::geteuid();
if ($get_id ne "0") {
print "You must be root to run this script!\n";
exit 1;
}

getopts('kfhco:g:');

sleep(30);

# This variable contains the commands necessary for this script to work properly
# If a user running this script is unable to run these commands, the user will be notified
# and the script will exit
our @run_commands = ("symmir","vxdg","vxvol","is_mounted");

#our @vx_diskgroups = ("db1","db2","db3","db4","db5","db6","db7","db8","db9","db10","db11","db12","db13","db14","db15","sys1","sys2","sys3","bdn1","bdn2","bdn3","bdn4","perfdb14","valeri","FREEwR1");
# dstage_temp disk group is temporary to create volumes that could not be replicated from prod - tkw 6/16/2008
#our @vx_diskgroups = ("db1","db2","db3","db4","db5","db6","db7","db8","db9","db10","db11","db12","db13","db14","db15","sys1","sys2","sys3","bdn1","bdn2","bdn3","bdn4","perfdb14","FREEwR1",);

#our @vx_diskgroups = ("db1","db2","db3","db4","db5","db6","db7","db8","db9","db10","db11","db12","db13","db14","db15","sys1","sys2","sys3","bdn1","bdn2","bdn3","bdn4","perfdb14",);
our @vx_diskgroups = ("db1","db2","db3","db4","db5","db6","db7","db8","db9","db10","db11","db12","db13","db14","db15","sys1","sys2","sys3","bdn1","bdn2","bdn3","bdn4");



# old diskgroup list..no longer valid as of 11/21/07
#
#our @vx_diskgroups = ("db1","db2","db3","db4","db5","db6","db7","db8","db9","db10","db11","db12","db13","db14","db15","FREEwR1","sys1","sys2","sys3");
# Directory that is created to contain temporary data to flush EMC cache
our $dummy_dir="offshoot";

# Directories used as source to create tar files in various directories to flush EMC cache
our @source_directories = ("/vbaperf/disk2/CDs2/","/vbaperf/disk2/audit/");

# Directories used as the target for tar files to flush EMC cache
#our @flush_directories = ("/tuxulog/","/vbadata/vetsnet/batch/","/tuxcompresslog/");
our @flush_directories = ("/devl/vnetbatch","/282arch1","/opt/patrol");

our $verify = "verify ";
our $config_file = "ARCH_BDN_DB_CFG";
our $symmetrix_id = "0411";
our $operation = "";
our $filesystem_list = "filesystems.txt";
our $bcv_command = "symmir -f $config_file -sid $symmetrix_id";
our $bcv_sec_command = "";
our %companion = ("restore" => "split",
"sync_split" => "split");
our @mounted_fs = {};
our $notify_operation = "";
our %email_not = ( "sync" => "sync",
"restore" => "restore",
"sync_split" => "sync and split"
);

# sync option commented to protect non-sysadmins "sync" => "-noprompt establish",
our %valid_ops = (
"restore" => "-noprompt restore",
"split" => "-noprompt split",
"sync_split" => "-noprompt establish",
"verify split" => "verify -split -bg",
"verify restore" => "verify -restored",
"verify sync_split" => "verify -synched",
"d_split" => "verify -synched",
"vximport" => "vximport",
"vxunmount" => "vxunmount"
);

# Log file
my $tee = new IO::Tee(">> /vbaperf/local/log/bcv_ops.log",\*STDOUT);

# Get current date
open (DT,"date |");
our $start_date =

;


# Define variable with termcap command to clear the screen
$terminal = Term::Cap->Tgetent( {OSPEED => 9600} );
#$clear_string = $terminal->Tputs('cl');
$clear_string = "";

# Clear screen
print $clear_string;


#######################
###* Begin program *###
#######################

# Main #

our $run_rc="./run_rc.ksh.mod";
$run_rc .= " cache" if defined $opt_c;

print $tee "-" x 50,"\n";
print "Starting ...\n";
print "CMD=$0 @ARGV\n\n";

validate_options();
check_cmd_scope();
check_process_state();
populate_filesystems();
execute_operation($opt_o);

if (defined($opt_c)) {
cache_flush();
}



## Subroutines ##

###################################
## check_process_state subroutine #
###################################
sub check_process_state() {
print $clear_string;
print "Checking for running applications...\n\n";
$check_command = "./check_procs.sh";


if (! -e $check_command) {
print $tee "Unable to locate the script(" . $check_command . ") to check for running applications!\n";
print $tee "Please replace the script and re-run this program.\n";
print $tee "If this is not possible, please manually verify that all aplications have been shutdown\n";
print $tee "and that all users are logged off that don't need to be logged on.\n";
print $tee "Then create a file called " . $check_command . " and put the following command inside it:\n";
print "exit 0;\n\n";
exit 1;
}

# Option to forcibly kill all processes that might interfere with a BCV operation
if (defined($opt_k)) {
$check_command = $check_command . " -f";
}

system($check_command);

if ($? ne 0) {
print $tee "There appear to be applications or application related processes still running!\n";
print $tee "Please verify that all applications have been properly shutdown.\n";
print $tee "A log indicating processes that were found has been generated and is named proc_log.txt\n";
exit 1;
}
print $tee "All applications appear to be shutdown.\n";
print $tee "Locking system to prevent further logins.\n";
print $tee "\nPlease wait...\n";

$lock_command = "touch /etc/nologin";
system ($lock_command);

sleep(5);
}

###################################
### validate_options subroutine ###
###################################
sub validate_options() {

# Check for options passed
if ((!defined($opt_o)) || (defined($opt_h))) {
print "Operation required!\n";
print "usage: bcv_mgmt.pl -o [restore|split|sync_split|vxunmount|vximport] -g [config file] -f -c -k(Forcibly kill all processes that might conflict with BCV operations)\n";
print "-c : Cache flush.\n";
print "-f : Add -full to BCV operation.\n";
print "-g : Specify custom BCV config file.\n";
print "-k : Forcibly kill all process that might interfere with a BCV operation.\n";
print "-o : BCV operation to perform.(required)\n";
exit 1;
}

if (defined($opt_g)) {
print $tee "You have requested a manual override of the default configuration file.\n";
print $tee "Verifying file...\n";
if (-e ($opt_g)) {
print $tee "File located successfully!\n\n";
print $tee "While this script has verified the existence of the file requested, it can not\n";
print $tee "verify that it contains valid devices. This is left to the end user to confirm prior\n";
print $tee "to running this scipt. If you are not sure that the file specified($opt_g) contains the\n";
print $tee "correct information, you should press control-c now to exit.\n";
print "\nPress any key to continue.\n";
#$prompt = ;
$config_file = $opt_g;
$bcv_command = "symmir -f $config_file -sid $symmetrix_id";
}
}

# Make sure we got valid options, if not inform user of valid options
if (defined($opt_o)) {
print $tee "Analyzing operation to be performed...\n\n";
$operation = $valid_ops{$opt_o};
if ($operation eq "") {
print $tee "Invalid operation requested!\n\n";
print $tee "Valid operations are: ";
foreach $ops (keys %valid_ops) {
print $tee $ops . " ";
}
print $tee "\n";
exit 1;
}
# Capture operation in notify string for later email notification
$notify_operation = $opt_o;
if (defined($opt_f)) {
print $tee "Requested operation is $opt_o with -full option\n\n";
if (defined($opt_c)) {
print $tee "This operation will be followed by a cache flush.\n";
}
#$bcv_command = "symmir -f $config_file -sid $symmetrix_id -full";
#$bcv_sec_command = "symmir -f $config_file -sid $symmetrix_id";
} else {
print $tee "Requested operation is $opt_o\n";
if (defined($opt_c)) {
print $tee "This operation will be followed by a cache flush.\n";
}
if (defined($opt_k)) {
print $tee "\nAdditionally all processes that might interfere with this BCV operation will be forcefully killed.\n";
}
}
}
sleep(5);
}
##################################
### check_cmd_scope subroutine ###
##################################
sub check_cmd_scope() {

# Verify access to needed commands to carry out requested operations
print $tee "Verifying PATH access to necessary commands\n";
foreach $cmd (@run_commands) {
open (COMMAND, "$cmd -H 2>&1 |");
@test_command = ;
$results_cmd_test = scalar(@test_command);
print $tee "cmd=$cmd (rc=$results_cmd_test)\n";
if ($results_cmd_test < 1) { print $tee "Unable to run the $cmd command!\n"; print $tee "This script should generally be run as root. Any other user must have $cmd in their path\n"; exit (1); } } print $tee "All commands verified successfully!\n"; print "\nPlease wait....\n"; sleep (5); print "Press any key to continue.\n"; #$prompt = ;
}

####################################
### execute_operation subroutine ###
####################################
sub execute_operation($) {
use Switch;
$operation = shift (@_);
print $clear_string;
print $tee "Checking prerequisites for the $operation operation\n\n";

switch ($operation) {
case "split" {
$cmd = "$bcv_command $valid_ops{\"$verify$operation\"} > /dev/null 2>&1";
system($cmd);
if ($? eq 0) {
print $tee "BCV's are already in a split state, please verify the operation you want to execute.\n";
exit 1;
}
check_mounted_fs();
check_vx_dgs();
print $tee "\nProceeding with the $operation operation at $start_date\n";
bcv_split($operation);
exit 0;
}
case "vximport" {
print $tee "Importing DG's!\n";
import_vx_dgs();
mount_fs();
exit 0;
}
case "vxunmount" {
print $tee "Unmounting filesystems!\n";
check_mounted_fs();
check_vx_dgs();
exit (0);
}
case "restore" {
print $tee `date`."start restore\n\n";
$cmd = "$bcv_command $valid_ops{\"$verify$operation\"} > /dev/null 2>&1";
print $tee "CMD=$cmd\n";
system($cmd);
if ($? == 0) {
print $tee "BCV's are already in a restored state, please verify the operation you want to execute.\n";
exit 1;
}
check_mounted_fs();
check_vx_dgs();
print $tee "\nProceeding with the $operation operation at $start_date\n";
bcv_restore($operation);
print $tee "\nStandard procedure calls for a split immediately after a restore, so the next step in the process will be a split.\n";
bcv_split($companion{"$operation"});
# exit 0;
import_vx_dgs();
mount_fs();
open (FDT,"date |");
our $finish_date = ;
print $tee "System recovered at $finish_date\nBeginning services startup.....\n";
$cmd_script = "$run_rc " . $email_not{"$notify_operation"};
print $tee "CMD=$cmd_script\n";
system($cmd_script);
$release_script = "rm /etc/nologin";
system($release_script);
print $tee "Releasing login lock file.\n";
}
case "sync_split" {
print $tee "Starting Sync/Split at $start_date\n";
$cmd = "$bcv_command $valid_ops{\"$verify$operation\"} > /dev/null 2>&1";
system($cmd);
if ($? == 0) {
print $tee "BCV's are already in a synchronized state, please verify the operation you want to execute.\n";
exit 1;
}
check_mounted_fs();
check_vx_dgs();
print $tee "\nProceeding with the $operation operation at $start_date\n";
bcv_sync($operation);
print $tee "\nStandard procedure calls for a split immediately after a sync, so the next step in the process will be a split.\n";
bcv_split($companion{"$operation"});
import_vx_dgs();
mount_fs();
open (FDT,"date |");
our $finish_date = ;
print $tee "System recovered at $finish_date\nBeginning services startup.....\n";
$cmd_script = "$run_rc " . $email_not{"$notify_operation"};
print $tee "CMD=$cmd_script\n";
system($cmd_script);
$release_script = "rm /etc/nologin";
system($release_script);
print $tee "Releasing login lock file.\n";
exit 0;
}
case "d_split" {
$start = 0;
while ($start != 1) {
$cmd="cd /vbaperf/refresh/EMC; symmir -f ALL_APPS_CFG -sid 1709 verify -synched";
system($cmd);
$return = $?/256;
print $tee "Status returned for sync: $return\n";
if ($return == 0) {
$start = 1;
}
sleep 10;
}
print $tee "Sync appears to have finished successfully!\n";
print "Sync appears to have finished successfully!\n";
print $tee "Beginning split!\n";
print "Beginning split!\n";
check_mounted_fs();
check_vx_dgs();
bcv_split("split");
import_vx_dgs();
mount_fs();
$cmd_script = "$run_rc ";
print $tee "CMD=$cmd_script\n";
system($cmd_script);
exit 0;
}
}
}


###################################
### check_mounted_fs subroutine ###
###################################
sub check_mounted_fs() {
print $tee "\nAttempting to unmount pertinent filesystems.......\n";
$mount_check = 1;
foreach $filesystem (@mounted_fs) {
chomp ($filesystem);
$cmd = "umount -f $filesystem > /dev/null 2>&1";
print $tee "$cmd\n";
system($cmd);

$cmd = "is_mounted $filesystem > /dev/null 2>&1";
system($cmd);
if ($? == 0) {
print $tee "Unable to unmount $filesystem!\n";
print $tee "Please verify that all processes associated with $filesystem have been terminated.\n";
print $tee "Then re-run this script.\n";
exit 1;
}
}
print $tee "All filesystems successfully unmounted.\n";
}

###############################
### check_vx_dgs subroutine ###
###############################
sub check_vx_dgs () {
print $tee "\nAttempting to deport all pertinent Veritas disk groups.......\n";
print $tee "Now deporting disk group: ";
foreach $diskgroup (@vx_diskgroups) {
print $tee "$diskgroup ";
$cmd = "vxdg deport $diskgroup > /dev/null 2>&1";
print $tee "$cmd\n";
system($cmd);
}
print $tee "\n";

# Verify that all pertinent diskgroups have been deported
foreach $diskgroup (@vx_diskgroups) {
$cmd = "vxdg -o verify list $diskgroup > /dev/null 2>&1";
system($cmd);
if ($? == 0) {
print $tee "Diskgroup: $diskgroup could not be deported. There may still be volumes in use for this diskgroup.\n";
print $tee "Please verify that all processes associated with volumes in $diskgroup have been terminated and re-run this script.\n";
print $tee "A list of all volumes in a diskgroup can be obtained by running the following command:\n";
print $tee "vxprint -g [diskgroup(e.g. $diskgroup)] -v -F \"%name\"\n";
exit 1;
}
}
print $tee `date`."All Veritas diskgroups successfully deported.\n";
}


#############################
### unmount_fs subroutine ###
#############################
sub unmount_fs () {
foreach $filesystem (@mounted_fs) {
$cmd = "umount -f $filesystem > /dev/null 2>&1";
system($cmd);
}
}


############################
### bcv_split subroutine ###
############################
sub bcv_split($) {
$operation = shift(@_);
if ($bcv_sec_command ne "") {
$bcv_command = $bcv_sec_command;
}
print $tee "\nThis process will take several minutes, consequently there will be no more output to the screen until the process is initiated.\n";
$cmd = "$bcv_command $valid_ops{$operation}\n";
print $tee "CMD=$cmd\n";
system($cmd);
print $tee "The split operation has been initiated and will take several minutes to complete.\n";
print $tee "This script will poll the symmetrix frame every 30 seconds to determine the current status of all BCV's.\n";
print $tee "Once the $operation is complete, this script will print out a status message indicating success.\n";

$cmd = "$bcv_command $valid_ops{\"$verify$operation\"} > /dev/null 2>&1";
print $tee "CMD=$cmd\n";
$bcv_status = 1;
while ($bcv_status == 1) {
system($cmd);
print $tee "$?...";

#if ($?/256 == 0) {
# $bcv_status = 0;
#} elsif ( $? =~ /\b(4|5|10|11|12|13|25|26|27|28|29|30)\b/) {
# sleep 30;
#} else {
# print $tee "Something is wrong!! EMC is acting funny! EXITING ...\n";
# exit 1;
#}

if ($?/256 == 0) {
$bcv_status = 0;
} else { sleep 30; }

}
print "\n";
print $tee `date`."The $operation operation is now complete!\n";
}

############################
### bcv_sync subroutine ###
############################
sub bcv_sync($) {
$operation = shift(@_);
print $tee "\nThe sync process will take several minutes, consequently there will be no more output to the screen until the process is initiated.\n";
$cmd = "$bcv_command $valid_ops{$operation}\n";
if ($opt_f) {
$cmd = "$bcv_command -full $valid_ops{$operation}\n";
}
print $tee "$cmd \n";
system($cmd);
print $tee "The $operation operation has been initiated and will take several minutes to complete.\n";
print $tee "This script will poll the symmetrix frame every 30 seconds to determine the current status of all BCV's.\n";
print $tee "Once the $operation is complete, this script will print out a status message indicating success.\n";

$cmd = "$bcv_command $valid_ops{\"$verify$operation\"} > /dev/null 2>&1";
print $tee "CMD=$cmd\n";
$bcv_status = 1;
while ($bcv_status == 1) {
system($cmd);
print $tee "$?...";

#if ($?/256 == 0) {
# $bcv_status = 0;
#} elsif ( $? =~ /\b(4|5|10|11|12|13|25|26|27|28|29|30)\b/) {
# sleep 30;
#} else {
# print $tee "Something is wrong!! EMC is acting funny! EXITING ...\n";
# exit 1;
#}

if ($?/256 == 0) {
$bcv_status = 0;
} else { sleep 30; }
}
print $tee "\nThe $operation operation is now complete!\n";
}


##############################
### bcv_restore subroutine ###
##############################
sub bcv_restore($) {
$operation = shift(@_);
print $tee "\nInitiating $operation process...\n";
print $tee "This process will take several minutes, consequently there will be no more output to the screen until the process is fully initiated.\n";
$cmd = "$bcv_command $valid_ops{$operation}\n";
if ($opt_f) {
$cmd = "$bcv_command -full $valid_ops{$operation}\n";
}
print $tee "CMD=$cmd\n";
system($cmd);
print $tee "The $operation operation has been initiated and will take several minutes to complete.\n";
print $tee "This script will poll the symmetrix frame every 30 seconds to determine the current status of all BCV's.\n";
print $tee "Once the $operation is complete, this script will print out a status message indicating success.\n";

# Check status of operation
$cmd = "$bcv_command $valid_ops{\"$verify$operation\"} > /dev/null 2>&1";
print $tee "CMD=$cmd\n";

$bcv_status = 1;
while ($bcv_status == 1) {
system($cmd);
print $tee "$?...";

#if ($?/256 == 0) {
# $bcv_status = 0;
#} elsif ( $? =~ /\b(4|5|10|11|12|13|25|26|27|28|29|30)\b/) {
# sleep 30;
#} else {
# print $tee "Something is wrong!! EMC is acting funny! EXITING ...\n";
# exit 1;
#}

if ($?/256 == 0) {
$bcv_status = 0;
} else { sleep 30; }

}
print $tee `date`."The $operation operation is now complete!\n";
}


#####################
## EMC Cache Flush ##
#####################
sub cache_flush() {
print $tee "\nPreparing files for cache flush....\n";

# Verify that critical directories are present to perform the flush
foreach $dir (@flush_directories) {
if (! -e $dir) {
print "Unable to locate directory $dir\nAborting cache flush!\n";
print "It may be necessary to mount this directory and then re-run this script.\n";
exit 1;
}
}
foreach $dir (@source_directories) {
if (! -e $dir) {
print "A necessary source directory($dir) is not present.\nThis directory is critical to the cache flush process.\nPlease make sure this directory is mounted and re-run this script.";
exit 1;
}
}
print $tee "All prerequisites for the cache flush have been met.\nThe cache flush process will take approximately 1 hour.\nNo BCV operations will occur until this process is complete.\n";
sleep 5;
print $tee "Starting flush process...";
foreach $source_dir (@source_directories) {
foreach $flush_dir (@flush_directories) {
chdir($flush_dir);
if (! -e $dummy_dir) { mkdir ($dummy_dir,oct(777)); }
chdir($dummy_dir);
$cmd = "/vbaperf/local/bin/tar -cf dummy.tar $source_dir > /dev/null 2>&1";
print $tee "CMD=$cmd\n";
system($cmd);
}
foreach $flush_dir (@flush_directories) {
chdir($flush_dir);
rmtree([$dummy_dir],0,0);
}
}
print $tee "Complete!\n";

# Change back to main program directory to continue execution
chdir($main_directory);

# Operational pause.
sleep 5;

# Clear screen
print $clear_string;

} # End subroutine cache_flush

######################
### recover_vx_dgs ###
######################
sub import_vx_dgs() {
print $tee "\nAttempting to import Veritas disk groups....\n";
print $tee "\nNow importing disk group: ";

foreach $diskgroup (@vx_diskgroups) {
print $tee "$diskgroup\n";
$cmd = "vxdg -fC import $diskgroup ";
print $tee "Executing '$cmd'\n";
system($cmd);
}

print $tee "\n";

# Verify that all pertinent diskgroups have been imported
foreach $diskgroup (@vx_diskgroups) {
$cmd = "vxdg -o verify list $diskgroup > /dev/null 2>&1";
print $tee "Executing $cmd \n";
system($cmd);
if ($? != 0) {
print $tee "Diskgroup: $diskgroup could not be imported. There may be disk problems associated with the disk group.\n";
print $tee "An administrator should be notified immediately!\n";
exit 1;
}
}
print $tee "All Veritas diskgroups successfully imported.\n";

print $tee "\nAttempting to start Veritas volumes....\n";
foreach $diskgroup (@vx_diskgroups) {
$cmd = "vxvol -g $diskgroup startall";
print $tee "Executing $cmd \n";
system($cmd);
if ($? != 0) {
print $tee "Could not start volumes for diskgroup $diskgroup!\n";
print $tee "Command issued was $cmd\n";
print $tee "There may be disk problems associated with the disk group.\n";
print $tee "An administrator should be notified immediately!\n";
exit 1;
}
}
print $tee "All Veritas volumes have been started!\n";
}

sub mount_fs() {
print $tee "\nAttempting to mount pertinent filesystems.......\n";
$mount_check = 1;
foreach $filesystem (reverse @mounted_fs) {
print $tee "Now mounting $filesystem\n";
$cmd = "mount $filesystem > /dev/null 2>&1";
system($cmd);
}

# Verify that all pertinent filesystems have been mounted
foreach $filesystem (@mounted_fs) {
$cmd = "is_mounted $filesystem > /dev/null 2>&1";
system($cmd);
if ($? != 0) {
print $tee "Unable to mount $filesystem!\n";
print $tee "Please verify that all processes associated with $filesystem have been terminated.\n";
print $tee "Then re-run this script.\n";
# satishs add on 10/10/07 to take care of nologin file
$release_script = "rm /etc/nologin";
system($release_script);
exit 1;
}
}
print $tee "All filesystems successfully mounted.\n";
print $tee "\n---------------------------------------\n";
print $tee "Any needed services may now be started.\n";
}

sub populate_filesystems() {
open (FS,"< $filesystem_list") or die ("Unable to open $filesystem_list, try copying $filesystem_list\.bak into place\n"); @mounted_fs = ;
}