#!/bin/ksh
# Script      : monitor_gg.sh
# Usage       : monitor_gg.sh ORACLE_SID GGHOME
# Example     : monitor_gg.sh DBNAME /u07/goldengate
#
# ******************
# Functions
# ******************
# Usage function.
function show_usage {
    echo " "
    echo "Usage: ./monitor_gg.sh ORACLE_SID $GGHOME"
    echo "   ORACLE_SID  : Database Instance. "
    echo "   GGHOME : Directory where GoldenGate is installed. "
    echo "Example: ./monitor_gg.sh DBNAME /u07/goldengate"
    echo " "
    exit 1
}
# **************************************
# Input parameter validation
# **************************************
if [ "$1" ]
then
   ORACLE_SID=`echo $1 | tr "[a-z]" "[A-Z]" `; export ORACLE_SID
else
   show_usage
fi
if [ "$2" ]
then
   GGHOME=`echo $2`; export GGHOME
else
   show_usage
fi
# **************************************
# Setting up the environment
# **************************************
export ORACLE_HOME=/u01/app/oracle/product/10.2.0/db_1
export PATH=$ORACLE_HOME/bin:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$GGHOME
export MAIL_LIST="support@networkcentricsupport.com"
# **************************************
# Gather GoldenGate Information
# **************************************
cd $GGHOME
$GGHOME/ggsci << EOF > /tmp/monitoring_gg.log
info all
exit
EOF
# ********************************************
# Monitoring GoldenGate processes
# ********************************************
cat /tmp/monitoring_gg.log | egrep 'MANAGER|EXTRACT|REPLICAT'| tr ":" " " | while read LINE
do
  case $LINE in
    *)
    PROCESS_TYPE=`echo $LINE | awk -F" " '{print $1}'`
    PROCESS_STATUS=`echo $LINE | awk -F" " '{print $2}'`
    if [ "$PROCESS_TYPE" == "MANAGER" ]
    then
       if [ "$PROCESS_STATUS" != "RUNNING" ]
       then
           SUBJECT="ALERT ... Goldengate process \"$PROCESS_TYPE\" is $PROCESS_STATUS on `uname -n`($ORACLE_SID)"
           mailx -s "$SUBJECT" $MAIL_LIST < $GGHOME/dirrpt/MGR.rpt
           exit 1
       else
           continue
       fi
    else
       PROCESS_NAME=`echo $LINE | awk -F" " '{print $3}'`
       if [ "$PROCESS_STATUS" != "RUNNING" ]
       then
           SUBJECT="ALERT ... Goldengate process \"$PROCESS_TYPE($PROCESS_NAME)\" is $PROCESS_STATUS on `uname -n`($ORACLE_SID)"
           mailx -s "$SUBJECT" $MAIL_LIST < $GGHOME/dirrpt/${PROCESS_NAME}.rpt
       fi
    fi
  esac
done
# **************************************
# Monitoring GoldenGate Error log
# **************************************
# The following warnings are ignored from the ggserr.log file
# OGG-00869: No unique key is defined for table ...
# OGG-01423: No valid default archive log destination directory found ...
# OGG-01842: CACHESIZE PER DYNAMIC DETERMINATION LESS THAN RECOMMENDED ...
# OGG-00938: Manager is stopping at user request
# OGG-00959: MINKEEPFILES option not used
# OGG-01003: Repositioning to rba ...
#
GG_ERROR_FILE=$GGHOME/ggserr.log
GG_ERROR_MNTR=$GGHOME/ggserr.monitor
GG_ERROR_DIFF=$GGHOME/ggserr.diff
touch $GG_ERROR_DIFF
if [[ -r ${GG_ERROR_FILE} ]]
then
    touch $GG_ERROR_MNTR
    cp $GG_ERROR_MNTR $GG_ERROR_MNTR".old"
    egrep "ERROR|WARNING" $GG_ERROR_FILE | egrep -v "OGG-00869|OGG-01423|OGG-01842|OGG-00938|OGG-00959|OGG-01003" > $GG_ERROR_MNTR
    CUR_CNT=`cat $GG_ERROR_MNTR |wc -l`
    PRV_CNT=`cat $GG_ERROR_MNTR".old" |wc -l`
    if [[ $CUR_CNT -lt $PRV_CNT ]]
    then
        # This means that ggserr.log is purged, so clear out the previous monitor results
        PRV_CNT=0
    fi
    if [[ $CUR_CNT -gt $PRV_CNT ]]
    then
        diff $GG_ERROR_MNTR $GG_ERROR_MNTR".old" | grep "^<" | sed -e 's/^< //' > ${GG_ERROR_DIFF}
        # send a mail..
        if test `cat ${GG_ERROR_DIFF} | wc -l` -gt 1
        then
            # multiple errors found. process only the last ten.
            tail -10 $GG_ERROR_DIFF > $GG_ERROR_DIFF".1"
        else
            # single error found.
            cp $GG_ERROR_DIFF $GG_ERROR_DIFF".1"
        fi
        SUBJECT="WARNING: Errors encountered in GoldenGate Replication on `uname -n` ($ORACLE_SID)"
        mailx -s "$SUBJECT" $MAIL_LIST < $GG_ERROR_DIFF".1"
    fi
    rm $GG_ERROR_DIFF*
fi