使用zabbix监控K8s(kubernetes)异常POD

通过监控异常的pod，及时发现k8s存在的问题。原理是使用kubectl get pod –all-namespaces，找到ready列，如果ready数量与desire数量不一致的，或者非RUNNING的状态POD,则认为这个pod异常，，进而告警。

配置agent监控项

/etc/zabbix/zabbix_agentd.d/k8s.conf

UserParameter=abnormal.pod, kubectl get pod --all-namespaces -o wide | awk 'NR>1{cmd="echo "$3" | bc";cmd|getline ret;close(cmd);if (ret != 1 || $4 != "Running"){print}}'

导入模板

<?xml version="1.0" encoding="UTF-8"?>
<zabbix_export>
    <version>3.2</version>
    <date>2017-06-16T03:51:42Z</date>
    <groups>
        <group>
            <name>Templates</name>
        </group>
    </groups>
    <templates>
        <template>
            <template>Template App K8s</template>
            <name>Template App K8s</name>
            <description/>
            <groups>
                <group>
                    <name>Templates</name>
                </group>
            </groups>
            <applications>
                <application>
                    <name>k8s</name>
                </application>
            </applications>
            <items>
                <item>
                    <name>abnormal pod</name>
                    <type>0</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>abnormal.pod</key>
                    <delay>60</delay>
                    <history>90</history>
                    <trends>0</trends>
                    <status>0</status>
                    <value_type>4</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>k8s</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
            </items>
            <discovery_rules/>
            <httptests/>
            <macros/>
            <templates/>
            <screens/>
        </template>
    </templates>
    <triggers>
        <trigger>
            <expression>{Template App K8s:abnormal.pod.strlen()}<>0</expression>
            <recovery_mode>0</recovery_mode>
            <recovery_expression/>
            <name>abnormal pod</name>
            <correlation_mode>0</correlation_mode>
            <correlation_tag/>
            <url/>
            <status>0</status>
            <priority>2</priority>
            <description/>
            <type>0</type>
            <manual_close>0</manual_close>
            <dependencies/>
            <tags/>
        </trigger>
    </triggers>
</zabbix_export>

应用模板

找到能执行kubectl的节点，添加k8s模板。