#!/bin/csh

# This script is designed to restart remote BS clients
#  who are reporting all purple in their status reports
#  (i.e. BS agent appears to have died or been killed).

# It has simple intelligence to watch for red, purple, or white
# status in the SSH column.  If this column reports
# any of these "bad" status messages, it will not try to restart
# the remote mcahine.

# You can set debug on to initially check the script - make sure
# it is doing what you expect!

# please note that you must have the SSH check running!

# I call this from crontab every 5 minutes, with an entry like this:
# 0,5,10,15,20,25,30,35,40,45,50,55 * * * * /home/bbuser/bin/remote_restart.csh >> /home/bbuser/logs/crontab.log 2>&1

# Standard disclaimers apply - please use at your own risk!

# Written 11.19.01 by Brent Parish
# Modified 02/20/02
# Version 1.3

# Dependancies:
# For the ssh to work, your $BBUSER must have permissions to ssh without password
# to all the clients.  This can be done through key exchange and $HOME/.ssh/known_hosts.
# An alternative is to replace ssh with rsh or similar mechanism

# # SET DEBUG = 1 or 2 to view debug messages in /home/bbuser/logs/crontab.log
set DEBUG = 0


# # DEFINE LOGNAME to be an empty file where you can log status/debug info
set LOGNAME = /home/bbuser/logs/crontab.log
# # OVERWRITE EVERYTIME (default):
echo '' > $LOGNAME
# # APPEND TO LOG (be carefull!!! This can get very big, especially with debug on!)
# # echo '' >> $LOGNAME


# # DEFINE THIS TO BE THE BS HTML DIR
set WEBDIR = /home/bbuser/www/html

cd $WEBDIR
# PULL TOGETHER LIST OF KNOWN CLIENTS
set list = `ls | cut -d'.' -f1 | sort -u`

if ($DEBUG == 2) then
   echo "DEBUG: Machine list found : $list" >> $LOGNAME
endif

foreach machine ($list)
   cd $WEBDIR
   # # FIGURE OUT HOW MANY CHECKS ARE SHOWING PURPLE FOR EACH CLIENT
   set check = `grep "purple" ${machine}* | wc -l`

   if ($DEBUG == 2) then
      set agents = `echo "$check / 2" | bc`
      echo "DEBUG: $machine shows $check purples ($agents agents failed)" >> $LOGNAME
   endif
	
   if ($check > 3) then	 
	# NOTE: There are two occurances of the string "purple"
	# 	for each monitored agent that fails to report
	#	So a value of 4 or greater = 2 failed agents

   	if ($DEBUG == 0) then
		echo "$machine" >> $LOGNAME
	endif
   	if ($DEBUG == 1) then
      	   set agents = `echo "$check / 2" | bc`
      	   echo "DEBUG: $machine shows $check purples ($agents agents failed)" >> $LOGNAME
   	endif

	if (-e $WEBDIR/${machine}.ssh.html) then
	   # Found a status file, going to check for purple flags (ssh down)

   	   if ($DEBUG > 0) then
      	      echo "DEBUG: $machine has an ssh status file:" >> $LOGNAME
	      ls -la $WEBDIR/${machine}.ssh.html >> $LOGNAME
   	   endif

   	   set check = `egrep 'COLOR="(purple|red|white)' ${machine}.ssh.html | wc -l`
	   if ($check > 0) then
	      # oops - server aparently is unavailable by ssh - cant restart

   	      if ($DEBUG > 0) then
      	         echo "DEBUG: status file for $machine has $check matches for string 'purple'" >> $LOGNAME
	         ls -la $WEBDIR/${machine}.ssh.html >> $LOGNAME
   	      endif

	      echo "`date` $machine needs to be restarted, but ssh connection is down" >> $LOGNAME
	   else
	      echo "`date` Restarting $machine via bbuser crontab after it failed to report agents" >> $LOGNAME
	      ssh -q -t bbuser@$machine '/home/bbuser/bin/bb_start restart;sleep 10' >> $LOGNAME
	   endif
	else
	   echo "`date` $machine needs to be restarted, but ssh connection is red" >> $LOGNAME
	endif
   endif
end
