監控系統 icinga (nagios 的分支) 分散式監控(Distributed Monitoring)篇

這邊的分散式監控是當主要的那台 (central) 負載太高，就可以把部份監控放在另一台上面 (slaver)，slaver 會把 check 後的結果傳回 central，我們仍可以在 central 查看最終的結果
比方說 central 負責監控所有 host 及 service，而 service 又分 http 、 smtp 、 cpu 、 mem 等等，我把 cpu 、mem 放在 slaver 處理，http、smtp 仍留在 central，但是 central 還是會看到 cpu、mem，只是把真正執行 check 的任務交給 slaver 再回傳結果，這個動作就是 passive，所以它是搭配 nsca 處理的

central

# 編譯 nsca
有特別設定 xxx-central，用於識別

yum install libmcrypt libmcrypt-devel 
wget https://downloads.sourceforge.net/project/nagios/nsca-2.x/nsca-2.9.1/nsca-2.9.1.tar.gz?r=&ts=1384410873&use_mirror=nchc
tar zxvf nsca-2.9.1.tar.gz
cd nsca-2.9.1
make clean; ./configure --with-nsca-user=icinga --with-nsca-grp=icinga --prefix=/usr/local/nsca-central
make all

# 複製設定 nsca

mkdir -p /usr/local/nsca-central/etc /usr/local/nsca-central/bin /usr/local/nsca-central/var
yes|cp -rp ./init-script /etc/init.d/nsca-central
yes|cp -rp ./src/nsca /usr/local/nsca-central/bin/nsca-central
yes|cp -rp ./sample-config/nsca.cfg /usr/local/nsca-central/etc/nsca-central.cfg
chmod +x /etc/init.d/nsca-central
chmod g+r /usr/local/nsca-central/etc/nsca-central.cfg
chown -R icinga.icinga /usr/local/nsca-central

# 在 icinga.cfg 加入設定
vi /usr/local/icinga/etc/icinga.cfg

check_external_commands=1
command_check_interval=-1
enable_notifications=1
accept_passive_service_checks=1
accept_passive_host_checks=1
log_passive_checks=1        # 用於測試，會有一堆 EXTERNAL COMMAND: PROCESS_SERVICE_CHECK_RESULT 記錄

# nsca 設定檔改一下
vi /usr/local/nsca-central/etc/nsca-central.cfg

pid_file=/var/run/nsca-central.pid
command_file=/usr/local/icinga/var/rw/icinga.cmd                # 改一下檔案名稱、路徑
alternate_dump_file=/usr/local/icinga/var/rw/nsca-central.dump  # 沒有 fifo 檔，此 dump 檔會自動產生附加寫入資料
password=icinga_nsca_sent                                       # 驗證碼，這邊設了 slaver 也要一樣

# nsca 啟動檔，改一下名稱、路徑
vi /etc/init.d/nsca-central

NscaBin=/usr/local/nsca-central/bin/nsca-central
NscaCfg=/usr/local/nsca-central/etc/nsca-central.cfg
LockFile=/var/lock/subsys/nsca-central
 
# See how we were called.
case "$1" in
  start)
        # Start daemons.
        echo -n "Starting nsca-central: "
        daemon $NscaBin -s -c $NscaCfg
        RETVAL=$?
        echo
        touch $LockFile
        ;;
  stop)
        # Stop daemons.
        echo -n "Shutting down nsca-central: "
        killproc nsca-central
        echo
        rm -f $LockFile
        ;;
  restart)
        $0 stop
        $0 start
        ;;
  status)
        status nsca-central
        ;;
  *)
        echo "Usage: nsca-central {start|stop|restart|status}"
        exit 1
esac
 
exit 0

再來讓 icinga define service 裡設定 active_checks_enabled 為 0 ，passive_checks_enabled 為 1
意思就是 central 不再對 cpu、mem 作執行 check 動作，只接受 slaver 過來的資料
vi cpu.cfg

active_checks_enabled           0
passive_checks_enabled          1

這樣子就可以

service icinga reload
service nsca-central start

再來

slaver

# 編譯 nsca
xxx-slaver

yum install libmcrypt libmcrypt-devel 
wget https://downloads.sourceforge.net/project/nagios/nsca-2.x/nsca-2.9.1/nsca-2.9.1.tar.gz?r=&ts=1384410873&use_mirror=nchc
tar zxvf nsca-2.9.1.tar.gz
cd nsca-2.9.1
make clean; ./configure --with-nsca-user=icinga --with-nsca-grp=icinga --prefix=/usr/local/nsca-slaver
make all

# 複製 nsca

mkdir -p /usr/local/nsca-slaver/etc /usr/local/nsca-slaver/bin /usr/local/nsca-slaver/var
yes|cp -rp ./src/send_nsca               "/usr/local/nsca-slaver/bin/send_nsca-slaver"
yes|cp -rp ./sample-config/send_nsca.cfg "/usr/local/nsca-slaver/etc/send_nsca-slaver.cfg"
chown -R icinga.icinga "/usr/local/nsca-slaver"

# 設定驗證碼
vi “/usr/local/nsca-slaver/etc/send_nsca-slaver.cfg”

password=icinga_nsca_sent # 驗證碼，與 central 相同

# 設定 icinga 要 send nsca command
vi /usr/local/icinga/etc/commands.d/submit_check_result_via_nsca.cfg

define command {
        command_name submit_check_result_via_nsca
        command_line /usr/local/icinga/libexec/eventhandlers/distributed-monitoring/submit_check_result_via_nsca $HOSTNAME$ '$SERVICEDESC$' $SERVICESTATEID$ '$SERVICEOUTPUT$|$SERVICEPERFDATA$'
}

# icinga 本身自帶的檔案
vi /usr/local/icinga/libexec/eventhandlers/distributed-monitoring/submit_check_result_via_nsca

# 修改 send_nsca 路徑
NscaBin="/usr/local/nsca-slaver/bin/send_nsca-slaver"
NscaCfg="/usr/local/nsca-slaver/etc/send_nsca-slaver.cfg"
# 修改 IcingaHost，就是 central 的 IP
IcingaHost="1.1.1.1"

再來讓 icinga define service 裡設定 active_checks_enabled 為 1 ，passive_checks_enabled 為 0
這是與 central 顛倒
vi cpu.cfg

active_checks_enabled           1
passive_checks_enabled          0  # 特別設 0 ，因為 slaver 也不需要

# 再修改 icinga.cfg
vi /usr/local/icinga/etc/icinga.cfg

enable_notifications=0                      # 讓 icinga 全部不作 notification
obsess_over_services=1                      # 我只要 service 作 send nsca
ocsp_command=submit_check_result_via_nsca   # 指令名稱，就是上面的 define command

# reload icinga

service icinga reload

# 在 slaver 測試丟資料到 central

/bin/echo -e "localhost\tdummy\t0\ttestesttest.\n" | /usr/local/nsca-slaver/bin/send_nsca-slaver -H 1.1.1.1 -c /usr/local/nsca-slaver/etc/send_nsca-slaver.cfg