Zabbix企业应用服务监控

常见企业应用监控

为满足企业需求,监控常见服务应用。需拓展自定义监控,监控服务如下:

  • 监控web应用(nginx)
  • 监控php应用
  • 监控数据库MySQL
  • 监控redis

自定义模板Nginx状态

用到的模板及脚本:zabbix.zip

开启监控页面访问测试

1
2
3
4
5
6
7
8
9
cat  /etc/nginx/conf.d/status.conf
server {
listen 80;
server_name localhost;
location /nginx_status {
stub_status;
access_log off;
}
}

准备Nginx监控脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
cat /etc/zabbix/zabbix_agentd.d/nginx_monitor.sh
#!/bin/bash
NGINX_COMMAND=$1
CACHEFILE="/tmp/nginx_status.txt"
CMD="/usr/bin/curl http://127.0.0.1/nginx_status/"
if [ ! -f $CACHEFILE ];then
$CMD >$CACHEFILE 2>/dev/null
fi
# Check and run the script
TIMEFLM=`stat -c %Y $CACHEFILE`
TIMENOW=`date +%s`

if [ `expr $TIMENOW - $TIMEFLM` -gt 60 ]; then
rm -f $CACHEFILE
fi
if [ ! -f $CACHEFILE ];then
$CMD >$CACHEFILE 2>/dev/null
fi

nginx_active(){
grep 'Active' $CACHEFILE| awk '{print $NF}'
exit 0;
}
nginx_reading(){
grep 'Reading' $CACHEFILE| awk '{print $2}'
exit 0;
}
nginx_writing(){
grep 'Writing' $CACHEFILE | awk '{print $4}'
exit 0;
}
nginx_waiting(){
grep 'Waiting' $CACHEFILE| awk '{print $6}'
exit 0;
}
nginx_accepts(){
awk NR==3 $CACHEFILE| awk '{print $1}'
exit 0;
}
nginx_handled(){
awk NR==3 $CACHEFILE| awk '{print $2}'
exit 0;
}
nginx_requests(){
awk NR==3 $CACHEFILE| awk '{print $3}'
exit 0;
}

case $NGINX_COMMAND in
active)
nginx_active;
;;
reading)
nginx_reading;
;;
writing)
nginx_writing;
;;
waiting)
nginx_waiting;
;;
accepts)
nginx_accepts;
;;
handled)
nginx_handled;
;;
requests)
nginx_requests;
;;
*)
echo 'Invalid credentials';
exit 2;
esac

编写zabbix agent监控配置文件

1
2
cat /etc/zabbix/zabbix_agentd.d/nginx_status.conf
UserParameter=nginx_status[*],/bin/bash /etc/zabbix/zabbix_agentd.d/nginx_monitor.sh $1

使用zabbix_get取值测试

1
2
[root@zabbix ~]# zabbix_get -s 10.1.1.20 -k nginx_status[accepts]
16

导入模板

链接模板查看数据

自定义模版监控php状态

开启监控页面

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
[root@web01 ~]# tail -1 /etc/php-fpm.d/www.conf    
pm.status_path = /php_status

[root@web01 ~]# cat /etc/nginx/conf.d/status.conf
server {
listen 80;
server_name localhost;
location /nginx_status {
stub_status;
access_log off;
}

location /php_status {
fastcgi_pass 127.0.0.1:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME html$fastcgi_script_name;
include fastcgi_params;
}
}

访问测试

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@web01 ]# curl 127.0.0.1/php_status         
pool: www
process manager: dynamic
start time: 04/Dec/2020:12:56:23 +0800
start since: 8
accepted conn: 1
listen queue: 0
max listen queue: 0
listen queue len: 128
idle processes: 4
active processes: 1
total processes: 5
max active processes: 1
max children reached: 0
slow requests: 0

准备访问脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
[root@web01 ~]# cat /etc/zabbix/zabbix_agentd.d/fpm.sh 
#!/bin/bash
##################################
# Zabbix monitoring script
#
# php-fpm:
# - anything available via FPM status page
#
##################################
# Contact:
# wang_xiansen0@163.com
##################################
# ChangeLog:
# 202001204
##################################

# Zabbix requested parameter
ZBX_REQ_DATA="$1"
ZBX_REQ_DATA_URL="$2"

# Nginx defaults
NGINX_STATUS_DEFAULT_URL="http://localhost/fpm/status"
WGET_BIN="/usr/bin/wget"

#
# Error handling:
# - need to be displayable in Zabbix (avoid NOT_SUPPORTED)
# - items need to be of type "float" (allow negative + float)
#
ERROR_NO_ACCESS_FILE="-0.91"
ERROR_NO_ACCESS="-0.92"
ERROR_WRONG_PARAM="-0.93"
ERROR_DATA="-0.94" # either can not connect / bad host / bad port

# Handle host and port if non-default
if [ ! -z "$ZBX_REQ_DATA_URL" ]; then
URL="$ZBX_REQ_DATA_URL"
else
URL="$NGINX_STATUS_DEFAULT_URL"
fi

# save the nginx stats in a variable for future parsing
NGINX_STATS=$($WGET_BIN -q $URL -O - 2>/dev/null)

# error during retrieve
if [ $? -ne 0 -o -z "$NGINX_STATS" ]; then
echo $ERROR_DATA
exit 1
fi

#
# Extract data from nginx stats
#
#RESULT=$(echo "$NGINX_STATS" | awk 'print $0;match($0, "^'"$ZBX_REQ_DATA"':[[:space:]]+(.*)", a) { print a[1] }')
#RESULT=$(echo "$NGINX_STATS" | grep "$ZBX_REQ_DATA" | awk -F : '{print $2}')
RESULT=$(echo "$NGINX_STATS" | awk -F : "{if(\$1==\"$ZBX_REQ_DATA\") print \$2}")
if [ $? -ne 0 -o -z "$RESULT" ]; then
echo $ERROR_WRONG_PARAM
exit 1
fi

echo $RESULT

exit 0

[root@web01 ~]# bash /etc/zabbix/zabbix_agentd.d/fpm.sh "total processes" http://127.0.0.1/php_status
5

准备agent配置文件

1
2
3
cat /etc/zabbix/zabbix_agentd.d/fpm.conf    
UserParameter=php-fpm[*],/etc/zabbix/zabbix_agentd.d/fpm.sh "$1" "$2"
[root@web01 ~]# systemctl restart zabbix-agent.service

zabbix_get取值测试

1
[root@zabbix ~]#  zabbix_get -s 10.1.1.20 -k php-fpm["total processes",http://127.0.0.1/php_status]

导入模板

导入之后需要修改一下模版里的宏配置

自定义模版监控redis状态

开启redis服务

1
redis -h 127.0.0.1 info

编写监控脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
vim /etc/zabbix/zabbix_agentd.d/redis.sh

#!/bin/bash
REDISPATH="/usr/local/bin/redis-cli"
HOST="127.0.0.1"
PORT="6379"
REDIS_PA="$REDISPATH -h $HOST -p $PORT info"
if [[ $# == 1 ]];then
case $1 in
cluster)
result=`$REDIS_PA|/bin/grep cluster|awk -F":" '{print $NF}'`
echo $result
;;
uptime_in_seconds)
result=`$REDIS_PA|/bin/grep uptime_in_seconds|awk -F":" '{print $NF}'`
echo $result
;;
connected_clients)
result=`$REDIS_PA|/bin/grep connected_clients|awk -F":" '{print $NF}'`
echo $result
;;
client_longest_output_list)
result=`$REDIS_PA|/bin/grep client_longest_output_list|awk -F":" '{print $NF}'`
echo $result
;;
client_biggest_input_buf)
result=`$REDIS_PA|/bin/grep client_biggest_input_buf|awk -F":" '{print $NF}'`
echo $result
;;
blocked_clients)
result=`$REDIS_PA|/bin/grep blocked_clients|awk -F":" '{print $NF}'`
echo $result
;;
#内存
used_memory)
result=`$REDIS_PA|/bin/grep used_memory|awk -F":" '{print $NF}'|awk 'NR==1'`
echo $result
;;
used_memory_human)
result=`$REDIS_PA|/bin/grep used_memory_human|awk -F":" '{print $NF}'|awk -F'K' '{print $1}'`
echo $result
;;
used_memory_rss)
result=`$REDIS_PA|/bin/grep used_memory_rss|awk -F":" '{print $NF}'`
echo $result
;;
used_memory_peak)
result=`$REDIS_PA|/bin/grep used_memory_peak|awk -F":" '{print $NF}'|awk 'NR==1'`
echo $result
;;
used_memory_peak_human)
result=`$REDIS_PA|/bin/grep used_memory_peak_human|awk -F":" '{print $NF}'|awk -F'K' '{print $1}'`
echo $result
;;
used_memory_lua)
result=`$REDIS_PA|/bin/grep used_memory_lua|awk -F":" '{print $NF}'`
echo $result
;;
mem_fragmentation_ratio)
result=`$REDIS_PA|/bin/grep mem_fragmentation_ratio|awk -F":" '{print $NF}'`
echo $result
;;
#rdb
rdb_changes_since_last_save)
result=`$REDIS_PA|/bin/grep rdb_changes_since_last_save|awk -F":" '{print $NF}'`
echo $result
;;
rdb_bgsave_in_progress)
result=`$REDIS_PA|/bin/grep rdb_bgsave_in_progress|awk -F":" '{print $NF}'`
echo $result
;;
rdb_last_save_time)
result=`$REDIS_PA|/bin/grep rdb_last_save_time|awk -F":" '{print $NF}'`
echo $result
;;
rdb_last_bgsave_status)
result=`$REDIS_PA|/bin/grep -w "rdb_last_bgsave_status" | awk -F':' '{print $2}' | /bin/grep -c ok`
echo $result
;;
rdb_current_bgsave_time_sec)
result=`$REDIS_PA|/bin/grep -w "rdb_current_bgsave_time_sec" | awk -F':' '{print $2}'`
echo $result
;;
#rdbinfo
aof_enabled)
result=`$REDIS_PA|/bin/grep -w "aof_enabled" | awk -F':' '{print $2}'`
echo $result
;;
aof_rewrite_scheduled)
result=`$REDIS_PA|/bin/grep -w "aof_rewrite_scheduled" | awk -F':' '{print $2}'`
echo $result
;;
aof_last_rewrite_time_sec)
result=`$REDIS_PA|/bin/grep -w "aof_last_rewrite_time_sec" | awk -F':' '{print $2}'`
echo $result
;;
aof_current_rewrite_time_sec)
result=`$REDIS_PA|/bin/grep -w "aof_current_rewrite_time_sec" | awk -F':' '{print $2}'`
echo $result
;;
aof_last_bgrewrite_status)
result=`$REDIS_PA|/bin/grep -w "aof_last_bgrewrite_status" | awk -F':' '{print $2}' | /bin/grep -c ok`
echo $result
;;
#aofinfo
aof_current_size)
result=`$REDIS_PA|/bin/grep -w "aof_current_size" | awk -F':' '{print $2}'`
echo $result
;;
aof_base_size)
result=`$REDIS_PA|/bin/grep -w "aof_base_size" | awk -F':' '{print $2}'`
echo $result
;;
aof_pending_rewrite)
result=`$REDIS_PA|/bin/grep -w "aof_pending_rewrite" | awk -F':' '{print $2}'`
echo $result
;;
aof_buffer_length)
result=`$REDIS_PA|/bin/grep -w "aof_buffer_length" | awk -F':' '{print $2}'`
echo $result
;;
aof_rewrite_buffer_length)
result=`$REDIS_PA|/bin/grep -w "aof_rewrite_buffer_length" | awk -F':' '{print $2}'`
echo $result
;;
aof_pending_bio_fsync)
result=`$REDIS_PA|/bin/grep -w "aof_pending_bio_fsync" | awk -F':' '{print $2}'`
echo $result
;;
aof_delayed_fsync)
result=`$REDIS_PA|/bin/grep -w "aof_delayed_fsync" | awk -F':' '{print $2}'`
echo $result
;;
#stats
total_connections_received)
result=`$REDIS_PA|/bin/grep -w "total_connections_received" | awk -F':' '{print $2}'`
echo $result
;;
total_commands_processed)
result=`$REDIS_PA|/bin/grep -w "total_commands_processed" | awk -F':' '{print $2}'`
echo $result
;;
instantaneous_ops_per_sec)
result=`$REDIS_PA|/bin/grep -w "instantaneous_ops_per_sec" | awk -F':' '{print $2}'`
echo $result
;;
rejected_connections)
result=`$REDIS_PA|/bin/grep -w "rejected_connections" | awk -F':' '{print $2}'`
echo $result
;;
expired_keys)
result=`$REDIS_PA|/bin/grep -w "expired_keys" | awk -F':' '{print $2}'`
echo $result
;;
evicted_keys)
result=`$REDIS_PA|/bin/grep -w "evicted_keys" | awk -F':' '{print $2}'`
echo $result
;;
keyspace_hits)
result=`$REDIS_PA|/bin/grep -w "keyspace_hits" | awk -F':' '{print $2}'`
echo $result
;;
keyspace_misses)
result=`$REDIS_PA|/bin/grep -w "keyspace_misses" | awk -F':' '{print $2}'`
echo $result
;;
pubsub_channels)
result=`$REDIS_PA|/bin/grep -w "pubsub_channels" | awk -F':' '{print $2}'`
echo $result
;;
pubsub_channels)
result=`$REDIS_PA|/bin/grep -w "pubsub_channels" | awk -F':' '{print $2}'`
echo $result
;;
pubsub_patterns)
result=`$REDIS_PA|/bin/grep -w "pubsub_patterns" | awk -F':' '{print $2}'`
echo $result
;;
latest_fork_usec)
result=`$REDIS_PA|/bin/grep -w "latest_fork_usec" | awk -F':' '{print $2}'`
echo $result
;;
connected_slaves)
result=`$REDIS_PA|/bin/grep -w "connected_slaves" | awk -F':' '{print $2}'`
echo $result
;;
master_link_status)
result=`$REDIS_PA|/bin/grep -w "master_link_status"|awk -F':' '{print $2}'|/bin/grep -c up`
echo $result
;;
master_last_io_seconds_ago)
result=`$REDIS_PA|/bin/grep -w "master_last_io_seconds_ago"|awk -F':' '{print $2}'`
echo $result
;;
master_sync_in_progress)
result=`$REDIS_PA|/bin/grep -w "master_sync_in_progress"|awk -F':' '{print $2}'`
echo $result
;;
slave_priority)
result=`$REDIS_PA|/bin/grep -w "slave_priority"|awk -F':' '{print $2}'`
echo $result
;;
#cpu
used_cpu_sys)
result=`$REDIS_PA|/bin/grep -w "used_cpu_sys"|awk -F':' '{print $2}'`
echo $result
;;
used_cpu_user)
result=`$REDIS_PA|/bin/grep -w "used_cpu_user"|awk -F':' '{print $2}'`
echo $result
;;
used_cpu_sys_children)
result=`$REDIS_PA|/bin/grep -w "used_cpu_sys_children"|awk -F':' '{print $2}'`
echo $result
;;
used_cpu_user_children)
result=`$REDIS_PA|/bin/grep -w "used_cpu_user_children"|awk -F':' '{print $2}'`
echo $result
;;
*)
echo "Usage:$0{uptime_in_seconds|connected_clients|client_longest_output_list|client_biggest_input_buf|blocked_clients|used_memory|used_memory_human|used_memory_rss|used_memory_peak|used_memory_peak_human|used_memory_lua|mem_fragmentation_ratio|rdb_changes_since_last_save|rdb_bgsave_in_progress|rdb_last_save_time|rdb_last_bgsave_status|rdb_current_bgsave_time_sec|aof_enabled|aof_rewrite_scheduled|aof_last_rewrite_time_sec|aof_current_rewrite_time_sec|aof_last_bgrewrite_status|aof_current_size|aof_base_size|aof_pending_rewrite|aof_buffer_length|aof_rewrite_buffer_length|aof_pending_bio_fsync|aof_delayed_fsync|rejected_connections|instantaneous_ops_per_sec|total_connections_received|total_commands_processed|expired_keys|evicted_keys|keyspace_hits|keyspace_misses|pubsub_channels|pubsub_patterns|latest_fork_usec|connected_slaves|master_link_status|master_sync_in_progress|master_last_io_seconds_ago|connected_slaves|slave_priority|used_cpu_user|used_cpu_sys|used_cpu_sys_children|used_cpu_user_children}"
;;
esac
#db0:key
elif [[ $# == 2 ]];then
case $2 in
keys)
result=`$REDIS_PA| /bin/grep -w "db0"| /bin/grep -w "$1" | /bin/grep -w "keys" | awk -F'=|,' '{print $2}'`
echo $result
;;
expires)
result=`$REDIS_PA| /bin/grep -w "db0"| /bin/grep -w "$1" | /bin/grep -w "expires" | awk -F'=|,' '{print $4}'`
echo $result
;;
avg_ttl)
result=`$REDIS_PA|/bin/grep -w "db0"| /bin/grep -w "$1" | /bin/grep -w "avg_ttl" | awk -F'=|,' '{print $6}'`
echo $result
;;
*)
echo "Usage:$0{db0 keys|db0 expires|db0 avg_ttl}"
;;
esac
fi

编辑agent配置文件

1
2
3
4
5
6
7
8
cat /etc/zabbix/zabbix_agentd.d/redis.conf 
#监控redis状态,我们可以根据这个参数对应的监控项创建redis状态触发器。
UserParameter=redis.status,/usr/local/bin/redis-cli -h 127.0.0.1 -p 6379 ping |grep -c PONG
#item参数如何get
UserParameter=redis_info[*], bash /etc/zabbix/zabbix_agentd.d/redis.sh $1 $2

#重启agentd
systemctl restart zabbix-agent

导入模板

查看监控状态

自定义模版监控MySQL状态

percona模版监控mysql

percona需要php环境

1
yum install php php-mysql -y

下载软件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
wget https://www.percona.com/downloads/percona-monitoring-plugins/percona-monitoring-plugins-1.1.8/binary/redhat/7/x86_64/percona-zabbix-templates-1.1.8-1.noarch.rpm
rpm -ivh percona-zabbix-templates-1.1.8-1.noarch.rpm

# 脚本目录 /var/lib/zabbix/percona/scripts
# 模板目录 /var/lib/zabbix/percona/templates

[root@zabbix /var/lib/zabbix/percona]# tree
.
├── scripts
│ ├── get_mysql_stats_wrapper.sh
│ └── ss_get_mysql_stats.php
└── templates
├── userparameter_percona_mysql.conf
└── zabbix_agent_template_percona_mysql_server_ht_2.0.9-sver1.1.8.xml

修改get_mysql_stats_wrapper.sh

修改get_mysql_stats_wrapper数据库登陆信息
第19行添加mysql账号密码:

1
2
[root@zabbix /var/lib/zabbix/percona/scripts ]# sed -n '19p' get_mysql_stats_wrapper.sh 
RES=`HOME=~zabbix mysql -uroot -p123456 -e 'SHOW SLAVE STATUS\G' | egrep '(Slave_IO_Running|Slave_SQL_Running):' | awk -F: '{print $2}' | tr '\n'

修改ss_get_mysql_stats.php

1
2
3
[root@zabbix /var/lib/zabbix/percona/scripts]# sed -n '30,31p' ss_get_mysql_stats.php 
$mysql_user = 'root';
$mysql_pass = '123456';

复制自定义监控项配置文件到zabbix目录

1
2
3
4
cd /var/lib/zabbix/percona/templates/
cp userparameter_percona_mysql.conf /etc/zabbix/zabbix_agentd.d/

systemctl restart zabbix-agent

zabbix_get取值测试

1
zabbix_get -s 10.1.1.60 -k MySQL.Sort-scan

导入模版

官方自带的模版有点问题,需要先装在2.x版本然后导出来,这里Linux Boy准备好模板了

http://download.boysec.cn/zabbix.zip