#!/bin/bash echo " " ## PATH export PATH=$PATH:/usr/local/mysql/bin failuretype=`cat /usr/local/orchestrator/log/detected_failuretype.log` planned_takeover=`cat /usr/local/orchestrator/log/planned_takeover.log` ## Failure Master Host IP failed_host_ip=`mysql --login-path=orchuser -sN -e "select ipv4 from orchestrator.hostname_ips where hostname='$1'"` ## Failure Master's Cluster Name failed_clustername=`mysql --login-path=orchuser -sN -e "select clustername from orchestrator.db_ha_vip where hostname='$1'"` ## Failure Master Host Ethernet Device failed_host_ethernet=`mysql --login-path=orchuser -sN -e "select net from orchestrator.db_ha_vip where hostname='$1'"` ## Promote Host IP promote_host_ip=`mysql --login-path=orchuser -sN -e "select ipv4 from orchestrator.hostname_ips where hostname='$2'"` ## Master VIP Address promote_master_vip=`mysql --login-path=orchuser -sN -e "select vip from orchestrator.db_ha_vip where clustername='$failed_clustername' and master_yn='Y'"` ## Promote Master VIP Subnet promote_master_subnet=`mysql --login-path=orchuser -sN -e "select subnet from orchestrator.db_ha_vip where hostname='$1'"` ## Promote Master Host Ethernet Device promote_host_ethernet=`mysql --login-path=orchuser -sN -e "select net from orchestrator.db_ha_vip where hostname='$2'"` ## Recovery - failover type 1 if [ "$failuretype" = "UnreachableMaster" ] then v_num_slave_hosts=`mysql --login-path=orchuser -sN -e "SELECT num_slave_hosts from orchestrator.database_instance where hostname='$2'"` ## Remove Master VIP # Master VIP Exists Check # 100% - not exists / 0% - exists master_vip_exists_chk=`ping -c 1 -W 1 $promote_master_vip | grep "packet loss" | awk '{print $6}'` if [ "$master_vip_exists_chk" = "0%" ] then /bin/ssh -o StrictHostKeyChecking=no $failed_host_ip /bin/sudo /sbin/ifconfig $failed_host_ethernet:0 down fi if [ "$v_num_slave_hosts" = "0" ] then ## Master VIP if-up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:0 $promote_master_vip netmask $promote_master_subnet up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /sbin/arping -c 5 -D -I $promote_host_ethernet -s $promote_master_vip $promote_master_vip else ## exist Slave VIP if-down bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:1 down ## Master VIP if-up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:0 $promote_master_vip netmask $promote_master_subnet up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /sbin/arping -c 5 -D -I $promote_host_ethernet -s $promote_master_vip $promote_master_vip fi fi ## relocate or failover if [ "$failuretype" = "DeadMaster" ] then ### take over if [ "$planned_takeover" = "planned_takeover" ] then ##### Master VIP <=> Slave VIP Relocate ##### ## save slave vip promote_host_owned_vip=`/bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /sbin/ifconfig $promote_host_ethernet:1 | grep inet | awk '{print $2}'` ## Remove Exists slave VIP /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:1 down ## Remove Exists master VIP /bin/ssh -o StrictHostKeyChecking=no $failed_host_ip /bin/sudo /sbin/ifconfig $failed_host_ethernet:0 down ## Master VIP if-up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:0 $promote_master_vip netmask $promote_master_subnet up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /sbin/arping -c 5 -D -I $promote_host_ethernet -s $promote_master_vip $promote_master_vip ## Slave VIP if-up /bin/ssh -o StrictHostKeyChecking=no $failed_host_ip /bin/sudo /sbin/ifconfig $failed_host_ethernet:1 $promote_host_owned_vip netmask $promote_master_subnet up /bin/ssh -o StrictHostKeyChecking=no $failed_host_ip /sbin/arping -c 5 -D -I $failed_host_ethernet -s $promote_host_owned_vip $promote_host_owned_vip ## Failover type 2 - ENV : 1master - 1 slave elif [ "$planned_takeover" = "" ] then v_num_slave_hosts=`mysql --login-path=orchuser -sN -e "SELECT num_slave_hosts from orchestrator.database_instance where hostname='$2'"` ## Remove Master VIP # Master VIP Exists Check # 100% - not exists / 0% - exists master_vip_exists_chk=`ping -c 1 -W 1 $promote_master_vip | grep "packet loss" | awk '{print $6}'` if [ "$master_vip_exists_chk" = "0%" ] then /bin/ssh -o StrictHostKeyChecking=no $failed_host_ip /bin/sudo /sbin/ifconfig $failed_host_ethernet:0 down fi if [ "$v_num_slave_hosts" = "0" ] then ## Master VIP if-up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:0 $promote_master_vip netmask $promote_master_subnet up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /sbin/arping -c 5 -D -I $promote_host_ethernet -s $promote_master_vip $promote_master_vip else ## exist Slave VIP if-down /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:1 down ## Master VIP if-up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /bin/sudo /sbin/ifconfig $promote_host_ethernet:0 $promote_master_vip netmask $promote_master_subnet up /bin/ssh -o StrictHostKeyChecking=no $promote_host_ip /sbin/arping -c 5 -D -I $promote_host_ethernet -s $promote_master_vip $promote_master_vip fi fi fi ## Table Update : Master/Slave VIP Host info mysql --login-path=orchuser -sN orchestrator -e "update db_ha_vip x join db_ha_vip y set x.vip=y.vip , y.vip=x.vip, x.master_yn=y.master_yn , y.master_yn=x.master_yn, x.chg_dt=now() , y.chg_dt=now() where x.hostname='$1' and y.hostname='$2'" /bin/cp /dev/null /usr/local/orchestrator/log/planned_takeover.log echo " "