global:
  # 经过此时间后，如果尚未更新告警，则将告警声明为已恢复。(即 prometheus 没有向 Alertmanager 发送告警了)
  resolve_timeout: 5m
  # 配置发送邮件信息
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_from: 'wangxiansen@boysec.cn'
  smtp_auth_username: 'wangxiansen@boysec.cn'
  smtp_auth_password: 'password'
  smtp_require_tls: false

# 读取告警通知模板的目录。
templates: 
- '/etc/alertmanager/template/*.tmpl'

# 所有报警都会进入到这个根路由下，可以根据根路由下的子路由设置报警分发策略
route:
  # 先解释一下分组，分组就是将多条告警信息聚合成一条发送，这样就不会收到连续的报警了。
  # 将传入的告警按标签分组(标签在 prometheus 中的 rules 中定义)，例如：
  # 接收到的告警信息里面有许多具有 cluster=A 和 alertname=LatencyHigh 的标签，这些个告警将被分为一个组。
  #
  # 如果不想使用分组，可以这样写group_by: [...]
  group_by: ['alertname', 'cluster', 'service']

  # 第一组告警发送通知需要等待的时间，这种方式可以确保有足够的时间为同一分组获取多个告警，然后一起触发这个告警信息。
  group_wait: 30s

  # 发送第一个告警后，等待"group_interval"发送一组新告警。
  group_interval: 5m

  # 分组内发送相同告警的时间间隔。这里的配置是每3小时发送告警到分组中。举个例子：收到告警后，一个分组被创建，等待5分钟发送组内告警，如果后续组内的告警信息相同,这些告警会在3小时后发送，但是3小时内这些告警不会被发送。
  repeat_interval: 3h 

  # 这里先说一下，告警发送是需要指定接收器的，接收器在receivers中配置，接收器可以是email、webhook、pagerduty、wechat等等。一个接收器可以有多种发送方式。
  # 指定默认的接收器
  receiver: team-X-mails
 
  # 下面配置的是子路由，子路由的属性继承于根路由(即上面的配置)，在子路由中可以覆盖根路由的配置
  # 下面是子路由的配置
  routes:
  # 使用正则的方式匹配告警标签
  - match_re:
      # 这里可以匹配出标签含有 service=foo1 或 service=foo2 或 service=baz 的告警
      service: ^(foo1|foo2|baz)$
    # 指定接收器为 team-X-mails
    receiver: team-X-mails
    # 这里配置的是子路由的子路由，当满足父路由的的匹配时，这条子路由会进一步匹配出 severity=critical 的告警，并使用 team-X-pager 接收器发送告警，没有匹配到的告警会由父路由进行处理。
    routes:
    - match:
        severity: critical
      receiver: team-X-pager

  # 这里也是一条子路由，会匹配出标签含有 service=files 的告警，并使用 team-Y-mails 接收器发送告警
  - match:
      service: files
    receiver: team-Y-mails
    # 这里配置的是子路由的子路由，当满足父路由的的匹配时，这条子路由会进一步匹配出 severity=critical 的告警，并使用 team-Y-pager 接收器发送告警，没有匹配到的会由父路由进行处理。
    routes:
    - match:
        severity: critical
      receiver: team-Y-pager

  # 该路由处理来自数据库服务的所有警报。如果没有团队来处理，则默认为数据库团队。
  - match:
      # 首先匹配标签service=database
      service: database
    # 指定接收器
    receiver: team-DB-pager
    # 根据受影响的数据库对告警进行分组
    group_by: [alertname, cluster, database]
    routes:
    - match:
        owner: team-X
      receiver: team-X-pager
      # 告警是否继续匹配后续的同级路由节点，默认false，下面如果也可以匹配成功，会向两种接收器都发送告警信息(猜测。。。)
      continue: true
    - match:
        owner: team-Y
      receiver: team-Y-pager
 
# 下面是关于inhibit(抑制)的配置，先说一下抑制是什么：抑制规则允许在另一个警报正在触发的情况下使一组告警静音。其实可以理解为告警依赖。比如一台数据库服务器掉电了，会导致db监控告警、网络告警等等，可以配置抑制规则如果服务器本身down了，那么其他的报警就不会被发送出来。

inhibit_rules:
#下面配置的含义：当有多条告警在告警组里时，并且他们的标签alertname,cluster,service都相等，如果severity: 'critical'的告警产生了，那么就会抑制severity: 'warning'的告警。
- source_match:  # 源告警(我理解是根据这个报警来抑制target_match中匹配的告警)
    severity: 'critical' # 标签匹配满足severity=critical的告警作为源告警
  target_match:  # 目标告警(被抑制的告警)
    severity: 'warning'  # 告警必须满足标签匹配severity=warning才会被抑制。
  equal: ['alertname', 'cluster', 'service']  # 必须在源告警和目标告警中具有相等值的标签才能使抑制生效。(即源告警和目标告警中这三个标签的值相等'alertname', 'cluster', 'service')
 
 
# 下面配置的是接收器
receivers:
# 接收器的名称、通过邮件的方式发送、
- name: 'team-X-mails'
  email_configs:
    # 发送给哪些人
  - to: 'team-X+alerts@example.org'
    # 是否通知已解决的警报
    send_resolved: true
 
# 接收器的名称、通过邮件和pagerduty的方式发送、发送给哪些人，指定pagerduty的service_key
- name: 'team-X-pager'
  email_configs:
  - to: 'team-X+alerts-critical@example.org'
  pagerduty_configs:
  - service_key: <team-X-key>
 
# 接收器的名称、通过邮件的方式发送、发送给哪些人
- name: 'team-Y-mails'
  email_configs:
  - to: 'team-Y+alerts@example.org'
 
# 接收器的名称、通过pagerduty的方式发送、指定pagerduty的service_key
- name: 'team-Y-pager'
  pagerduty_configs:
  - service_key: <team-Y-key>
 
# 一个接收器配置多种发送方式
- name: 'ops'
  webhook_configs:
  - url: 'http://prometheus-webhook-dingtalk.kube-ops.svc.cluster.local:8060/dingtalk/webhook1/send'
    send_resolved: true
  email_configs:
  - to: 'wangxiansen@boysec.cn'
    send_resolved: true
  - to: 'admin@boysec.cn'
    send_resolved: true

groups:
- name: node-exporter.rules   
  rules:
  - record: instance:node_num_cpu:sum
    expr: |
      count without (cpu, mode) (
        node_cpu_seconds_total{job="node-exporter",mode="idle"}
      ) 

groups:
- name: test
  rules:
  - alert: NodeFilesystemAlmostOutOfSpace
    expr: node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 10
    for: 30m
    labels:
      severity: warning
    annotations:
      description: ' {{ $labels.instance }} 节点 {{ $labels.device }} 文件系统剩余空间: {{ printf "%.2f" $value }}% '
      summary: '文件系统剩余空间不足 10%'

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
  name: demo
  namespace: monitoring
spec:
  groups:
  - name: group1
    rules:
    - alert: alert1
      annotations:
        description: alert-1
        summary: alert-1
      expr: up == 0
      for: 15m
      labels:
        severity: critical
     - alert: alert2
......
  - name: group2
    rules:
    - alert: alert3
......

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: demo
  namespace: monitoring
spec:
  groups:
  - name: demo
    rules:
    - alert: nodeDiskUsage
      annotations:
        description: |
          节点 {{$labels.instance }}
          挂载目录 {{ $labels.mountpoint }}
          当前可用空间 {{ printf "%.2f" $value }}%          
        summary: |
                    挂载目录可用空间低于 50%
      expr: |
        node_filesystem_avail_bytes{fstype!="",job="node-exporter"} /
        node_filesystem_size_bytes{fstype!="",job="node-exporter"} * 100 < 50        
      for: 1m
      labels:
        severity: warning

# vim alertmanager.yaml
global:
  resolve_timeout: 5m
  smtp_from: 'wang_xiansen0@163.com'
  smtp_smarthost: 'smtp.163.com:25'
  smtp_auth_username: 'wang_xiansen0@163.com'
  smtp_auth_password: 'password'
  smtp_require_tls: false
  smtp_hello: '163.com'

templates:
  - '/etc/alertmanager/configmaps/alertmanager-templates/*.tmpl'

route:
  receiver: Default
  group_by: ['alertname', 'cluster']
  continue: false
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
inhibit_rules:
- source_matchers:
  - severity="critical"
  target_matchers:
  - severity=~"warning|info"
  equal:
  - namespace
  - alertname
- source_matchers:
  - severity="warning"
  target_matchers:
  - severity="info"
  equal:
  - namespace
  - alertname
- source_matchers:
  - alertname="InfoInhibitor"
  target_matchers:
  - severity="info"
  equal:
  - namespace
receivers:
- name: Default
  email_configs:
  - to: 'wangxiansen@boysec.cn'
    send_resolved: true

# vim alertmanagerconfig.yaml
apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: email-config
  namespace: monitoring
  labels:
    alertmanagerConfig: wangxiansen
spec:
  route:
    groupBy: ['alertname']
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 12h
    receiver: 'Critical'
    continue: false
    routes:
      - receiver: 'Critical'
        match:
          severity: critical
  receivers:
    - name: Critical
      emailConfigs:
        - to: 'wangxiansen@boysec.cn'
          sendResolved: true
      webhookConfigs:
        - url: http://dingtalk
          sendResolved: true

# vim alertmanager-alertmanager.yaml
apiVersion: monitoring.coreos.com/v1
kind: Alertmanager
metadata:
  labels:
    app.kubernetes.io/component: alert-router
    app.kubernetes.io/instance: main
    app.kubernetes.io/name: alertmanager
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 0.26.0
  name: main
  namespace: monitoring
spec:
  image: quay.io/prometheus/alertmanager:v0.26.0
  nodeSelector:
    kubernetes.io/os: linux
  podMetadata:
    labels:
      app.kubernetes.io/component: alert-router
      app.kubernetes.io/instance: main
      app.kubernetes.io/name: alertmanager
      app.kubernetes.io/part-of: kube-prometheus
      app.kubernetes.io/version: 0.26.0
  replicas: 1                 # 资源问题，这里就先启动一个pods
  resources:
    limits:
      cpu: 100m
      memory: 100Mi
    requests:
      cpu: 4m
      memory: 100Mi
  secrets: []
  alertmanagerConfigSelector: # 匹配 AlertmanagerConfig 的标签
    matchLabels:
      alertmanagerConfig: wangxiansen
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: alertmanager-main
  version: 0.26.0

# vim alertmanager-alertmanager.yaml
apiVersion: monitoring.coreos.com/v1
kind: Alertmanager
metadata:
spec:
.....
  alertmanagerConfigSelector:
    matchLabels:
      alertmanager: main
  configMaps:
  - alertmanager-templates

apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: email-config
  namespace: monitoring
  labels:
    alertmanagerConfig: wangxiansen
spec:
  route:
    groupBy: ['alertname']
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 12h
    receiver: 'Critical'
    continue: false
    routes:
      - receiver: 'Critical'
        match:
          severity: critical
  receivers:
    - name: Critical
      emailConfigs:
        - to: 'wangxiansen@boysec.cn'
          html: '{{ template "email.html" . }}'          # 添加 与模板中的 define 对应
          sendResolved: true

target_match:
  [ <labelname>: <labelvalue>, ... ]

target_match_re:
  [ <labelname>: <regex>, ... ]

target_matchers:
  [ - <matcher> ... ]


source_match:
  [ <labelname>: <labelvalue>, ... ]

source_match_re:
  [ <labelname>: <regex>, ... ]

source_matchers:
  [ - <matcher> ... ]

[ equal: '[' <labelname>, ... ']' ]

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: node
  namespace: default
spec:
  groups:
  - name: node-mem
    rules:
    - alert: NodeMemoryUsage
      annotations:
        description: '{{$labels.instance}}: 内存使用率高于 30% (当前值为: {{ printf "%.2f" $value }}%)'
        summary: '{{$labels.instance}}: 检测到高内存使用率'
      expr: |
        (node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)) /
        node_memory_MemTotal_bytes * 100 > 30       
      for: 1m
      labels:
        team: node
        severity: critical
  - name: node-load
    rules:
    - alert: NodeLoad
      annotations:
        summary: '{{ $labels.instance }}: 低节点负载检测'
        description: '{{ $labels.instance }}: 节点负载低于 1 (当前值为: {{ $value }})'
      expr: node_load5 < 1
      for: 2m
      labels:
        team: node
        severity: normal

# vim alertmanagerconfig.yaml
apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: email-config
  namespace: monitoring
  labels:
    alertmanagerConfig: wangxiansen
spec:
  route:
    groupBy: ['alertname']
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 12h
    receiver: 'Critical'
    continue: false
    routes:
      - receiver: 'Critical'
        match:
          severity: critical
  receivers:
    - name: Critical
      emailConfigs:
        - to: 'wangxiansen@boysec.cn'
          html: '{{ template "email.html" . }}'
          sendResolved: true
  inhibitRules:
  - equal: ['instance']
    sourceMatch:                  # 最底层的报警规则（相当于节点）
    - name: alertname             # 通过 alertname 名称匹配
      value: NodeMemoryUsage      # 对报警名称为NodeMemoryUsage 抑制
    - name: severity
      value: critical
    targetMatch:                  # 上层节点报警（相当于节点服务）
    - name: severity
      value: normal