prometheus

架构图

prometheus-server

config/prometheus.yml

1
2
3
4
5
6
7
8
9
10
11
global:
  scrape_interval:     60s
  evaluation_interval: 60s

rule_files:
  - "/usr/local/prometheus/rules/*.rules"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
    - targets: ["127.0.0.1:9090"]

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#!/bin/bash
echo "Asia/Shanghai" > /etc/timezone

mkdir -p $(pwd)/data
chown -R 65534.65534 $(pwd)/data

port=9090

docker stop prometheus
docker rm prometheus
docker run -d --net host \
--name prometheus \
-v /etc/timezone:/etc/timezone:ro \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro \
-v $(pwd)/config/:/etc/prometheus/ \
-v $(pwd)/data/:/prometheus/ \
-v $(pwd)/groups/:/usr/local/prometheus/groups/ \
-v $(pwd)/rules/:/usr/local/prometheus/rules/ \
prom/prometheus:v2.46.0 \
--config.file=/etc/prometheus/prometheus.yml \
--web.console.libraries=/etc/prometheus/console_libraries \
--web.console.templates=/etc/prometheus/consoles \
--storage.tsdb.path=/prometheus \
--storage.tsdb.retention.time=60d \
--web.enable-admin-api

exporter

postgres_exporter

单个数据库

setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#!/bin/bash

start_pg_exporter() {
pg_host=$1
pg_port=$2
pg_user=$3
pg_password=$4
exporter_port=$5
docker kill postgresql-exporter-${exporter_port}
docker rm postgresql-exporter-${exporter_port}
docker run -d \
--restart=always \
-p ${exporter_port}:9187 \
--name postgresql-exporter-${exporter_port} \
-v /etc/localtime:/etc/localtime:ro \
-v /etc/timezone:/etc/timezone:ro \
-e DATA_SOURCE_NAME="postgresql://${pg_user}:${pg_password}@${pg_host}:${pg_port}/postgres?sslmode=disable" \
wrouesnel/postgres_exporter
}

pgid="hdy-nmg-server-001"
pg_host="10.0.18.2"
pg_port="5432"
pg_user="postgres"
pg_password=""
exporter_port=13101
start_pg_exporter ${pg_host} ${pg_port} ${pg_user} ${pg_password} ${exporter_port}
配置增加
1
2
3
4
5
6
7
# hdy-nmg-server-001
- job_name: 'hdy-nmg-server-001'
  static_configs:
    - targets: ['127.0.0.1:13101']
      labels:
        type: postgresql
        instance: hdy-nmg-server-001

多个数据库

setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#!/bin/bash

mkdir -p config/
cat << EOF > config/postgres_exporter.yml
auth_modules:
  hdy-nmg-server-001:
    type: userpass
    userpass:
      username: postgres
      password: NoKgWU
    options:
      sslmode: disable
  aliyun-hz-server-master:
    type: userpass
    userpass:
      username: postgres
      password: 9VczV
    options:
      sslmode: disable
EOF

docker kill postgresql-exporter
docker rm postgresql-exporter
docker run -d --net=host \
--name postgresql-exporter \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
--restart=always \
-v $(pwd)/config/:/etc/postgres-exporter/ \
quay.io/prometheuscommunity/postgres-exporter \
--config.file /etc/postgres-exporter/postgres_exporter.yml \
--web.listen-address=:13101
配置增加
1
2
3
4
5
6
7
8
9
10
11
12
13
- job_name: 'postgresql-exporter'
  scrape_interval: 15s
  scrape_timeout: 10s
  file_sd_configs:
    - files: ['/usr/local/prometheus/groups/postgresql/*.json','/usr/local/prometheus/groups/postgresql/*.yml']
  metrics_path: /probe
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [instance]
      target_label: __param_auth_module
    - target_label: __address__
      replacement: "10.0.18.2:13101"
groups/postgresql/postgresql.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
- targets: [ "10.0.18.2:5432"]
  labels:
    job_name: hdy-nmg-server-001
    instance: hdy-nmg-server-001
    saltid: hdy-nmg-server-001
    comment: "hdy-nmg-server-001"

- targets: [ "47.xxx.xxx.xxx:5432"]
  labels:
    job_name: aliyun-hz-server-master
    instance: aliyun-hz-server-master
    saltid: aliyun-hz-server-master
    comment: "aliyun-hz-server-master"

mysql_exporter

单个数据库

setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/bin/bash

# hdy-nmg-server-001

port=10309

docker kill mysqld-exporte-${port}
docker rm mysqld-exporte-${port}

cat << EOF > .my.cnf
[client]
user=root
password=
EOF

docker run -d --name mysqld-exporte-${port} \
--net host \
-v /etc/timezone:/etc/timezone:ro \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro \
-v $(pwd)/.my.cnf:/config/.my.cnf \
prom/mysqld-exporter:v0.15.0 \
--web.listen-address=:${port} \
--config.my-cnf="/config/.my.cnf"
配置增加
1
2
3
4
5
6
7
# hdy-nmg-server-001
- job_name: 'mysql'
  static_configs:
    - targets: ['127.0.0.1:10309']
      labels:
        type: mysql
        instance: hdy-nmg-server-001

多个数据库

setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#!/bin/bash

# hdy-nmg-server-001

port=10309

docker kill mysqld-exporte-${port}
docker rm mysqld-exporte-${port}

cat << EOF > .my.cnf
[hdy-nmg-server-001]
user=root
password=
[aliyun-hz-server-master]
user=root
password=
EOF

docker run -d --name mysqld-exporte-${port} \
--net host \
-v /etc/timezone:/etc/timezone:ro \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro \
-v $(pwd)/.my.cnf:/config/.my.cnf \
prom/mysqld-exporter:v0.15.0 \
--web.listen-address=:${port} \
--config.my-cnf="/config/.my.cnf"
配置增加
1
2
3
4
5
6
7
8
9
10
11
12
13
- job_name: 'mysql-exporter'
  scrape_interval: 15s
  scrape_timeout: 10s
  file_sd_configs:
    - files: ['/usr/local/prometheus/groups/mysql/*.json','/usr/local/prometheus/groups/mysql/*.yml']
  metrics_path: /probe
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [instance]
      target_label: __param_auth_module
    - target_label: __address__
      replacement: "10.0.18.2:10309"
groups/mysql/mysql.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
- targets: [ "10.0.18.2:3306"]
  labels:
    job_name: hdy-nmg-server-001
    instance: hdy-nmg-server-001
    saltid: hdy-nmg-server-001
    comment: "hdy-nmg-server-001"

- targets: [ "47.xxx.xxx.xxx:3306"]
  labels:
    job_name: aliyun-hz-server-master
    instance: aliyun-hz-server-master
    saltid: aliyun-hz-server-master
    comment: "aliyun-hz-server-master"

blackbox_exporter

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
#!/bin/bash

docker stop blackbox_exporter
docker rm blackbox_exporter
docker run -d \
  --restart=always \
  -p 9115:9115 \
  --name blackbox_exporter \
  -v $(pwd)/config:/config \
  -v /etc/timezone:/etc/timezone:ro \
  -v /etc/localtime:/etc/localtime:ro \
  quay.io/prometheus/blackbox-exporter:latest --config.file=/config/blackbox.yml

config/blackbox.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
modules:
  # 通用get
  http_2xx:
    prober: http
    timeout: 35s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: []  # Defaults to 2xx
      method: GET
      preferred_ip_protocol: "ip4" # defaults to "ip6"
      ip_protocol_fallback: false  # no fallback to "ip6"
  # 无参post
  http_post_2xx:
    prober: http
    timeout: 35s
    http:
      method: POST
      headers:
        Content-Type: application/json

配置增加

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
- job_name: 'blackbox-exporter'
  scrape_interval: 60s
  scrape_timeout: 35s
  file_sd_configs:
    - files: ['/usr/local/prometheus/groups/blackbox/*.json','/usr/local/prometheus/groups/blackbox/*.yml']
  metrics_path: /probe
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [module]
      target_label: __param_module
    - source_labels: [address]
      target_label: address
    - target_label: __address__
      replacement: "127.0.0.1:9115"

groups/blackbox/blackbox.yml

1
2
3
4
5
- targets: [ "http://10.0.18.2:8008/api/video_type/list" ]
  labels:
    module: "http_2xx"
    address: "http://10.0.18.2:8008/api/video_type/list"
    instance: "wx-video"

grafana

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/bash
echo 'Asia/Shanghai' > /etc/timezone

port=3000

mkdir -p data/ && chmod 777 data/ && chown -R 472.472 data/

docker kill grafana
docker rm grafana
docker run -d --name=grafana \
--restart=always \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
-v $(pwd)/data/:/var/lib/grafana/ \
-p ${port}:3000 \
grafana/grafana:10.0.3

数据源

添加Prometheus，修改HTTP中的url为http://10.0.18.2:9090

仪表盘

在https://grafana.com/grafana/dashboards/中寻找模板

推荐模板：

在mysql的仪表盘中可以添加一个监控项

mysql_version_info{instance="$host"}

options中选择instant，legend填写{{version}}

右侧选择stat,Text mode选择Name

推送

prometheus-webhook-dingtalk

config/config.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# Request timeout
timeout: 5s

# Customizable templates path
templates:
  - templates/alertmanager-dingtalk.tmpl

targets:
  webhook:
    # 运维内部群机器人
    url: https://oapi.dingtalk.com/robot/send?access_token=1adaa314f6d04b7
    # secret for signature
    secret: SEC9e23
    message:
      text: '{{ template "dingtalk.to.message" . }}'

templates/alertmanager-dingtalk.tmpl

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
{{ define "dingtalk.to.message" }}

{{- if gt (len .Alerts.Firing) 0 -}}
{{- range $index, $alert := .Alerts -}}

=========  **监控告警** =========  

**告警程序:**     Alertmanager  
**告警类型:**    {{ $alert.Labels.alertname }}    
**告警级别:**    {{ $alert.Labels.severity }}    
**告警状态:**    {{ .Status }}    
**故障主机:**    {{ $alert.Labels.instance }} {{ $alert.Labels.device }}    
**告警主题:**    {{ .Annotations.summary }}     
**告警详情:**    {{ $alert.Annotations.message }}{{ $alert.Annotations.description}}    
**告警图形:**    [📈]({{ .GeneratorURL }})   
**主机标签:**    {{ range .Labels.SortedPairs  }}  </br> [{{ .Name }}: {{ .Value | markdown | html }} ]
{{- end }} </br>

**故障时间:**    {{ $alert.StartsAt.Local.Format "2006-01-02 15:04:05" }}     

========= = end =  =========  
{{- end }}
{{- end }}

{{- if gt (len .Alerts.Resolved) 0 -}}
{{- range $index, $alert := .Alerts -}}

========= 告警恢复 =========  
**告警程序:**     Alertmanager   
**告警主题:**    {{ $alert.Annotations.summary }}  
**告警主机:**    {{ .Labels.instance }}   
**告警类型:**    {{ .Labels.alertname }}  
**告警级别:**    {{ $alert.Labels.severity }}   
**告警状态:**    {{   .Status }}    
**告警详情:**    {{ $alert.Annotations.message }}{{ $alert.Annotations.description}}        
**告警图形:**    [📈]({{ .GeneratorURL }})   
**故障时间:**    {{ $alert.StartsAt.Local.Format "2006-01-02 15:04:05" }}     
**恢复时间:**    {{ $alert.EndsAt.Local.Format "2006-01-02 15:04:05" }}   

========= = **end** =  =========    
{{- end }}
{{- end }}
{{- end }}

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#!/bin/bash

version="v2.1.0"

docker stop prometheus-webhook-dingtalk
docker rm prometheus-webhook-dingtalk
docker run -d \
--net host \
--name prometheus-webhook-dingtalk \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
-v $(pwd)/config/:/etc/prometheus-webhook-dingtalk/ \
-v $(pwd)/templates/:/prometheus-webhook-dingtalk/templates/ \
timonwong/prometheus-webhook-dingtalk:${version} \
--web.listen-address=:8060 \
--web.enable-ui \
--web.enable-lifecycle \
--config.file=/etc/prometheus-webhook-dingtalk/config.yml \
--log.level=info \
--log.format=logfmt \

alertmanager

config/alertmanager.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
global:
  # 每2分钟检查一次是否恢复
  resolve_timeout: 2m
# route用来设置报警的分发策略

route:
  receiver: 'ops-dingtalk'
  group_by: ['...']
  group_wait: 3s
  group_interval: 1m
  repeat_interval: 5m
  routes:
    - receiver: 'ops-dingtalk'
      group_by: ['...']
      # 等待时间，如果同一个组有新的告警会被合并到同一个消息内
      group_wait: 3s
      # 异常持续报警间隔时间为 group_interval + repeat_interval 总和
      # group_interval 相同的Group之间发送告警通知的时间间隔
      group_interval: 1m
      # 一条成功发送的告警，在最终发送通知之前的等待时间
      repeat_interval: 5m
      matchers:
      - severity=~"^信息$|^警告$|^一般严重$|^严重$|^灾难$|^测试模板$"
receivers:
  - name: 'ops-dingtalk'  
    webhook_configs:
      - send_resolved: true
        url: 'http://10.0.18.2:8060/dingtalk/webhook/send'
        max_alerts: 0

templates/alertmanager-email.tmpl

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
#########基于alertmanager官方模板修改，内容可删减#########

{{ define "__alertmanager" }}Alertmanager{{ end }}

{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__description" }}{{ end }}

{{ define "email.to.html" }}
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!--
Style and HTML derived from https://github.com/mailgun/transactional-email-templates

The MIT License (MIT)

Copyright (c) 2014 Mailgun

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-->
<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<head style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<meta name="viewport" content="width=device-width" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
<title style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{{ template "__subject" . }}</title>

</head>

<body itemscope="" itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; width: 100% !important; background-color: #f6f6f6; margin: 0; padding: 0;" bgcolor="#f6f6f6">

<table style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
  <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 0;">
    <table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff">
      <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #E6522C; margin: 0; padding: 20px;" align="center" bgcolor="#E6522C" valign="top">
          发生 {{ .Alerts | len }}  个 {{ range .GroupLabels.SortedPairs }}
                {{ .Value }}
          {{ end }} 告警 ！！请尽快处理
        </td>
      </tr>
      <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 10px;" valign="top">
          <table border="1" cellpadding="2" cellspacing="0" width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
            <tr border="1" cellpadding="2" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
            </tr>
          <table border="1" cellpadding="2" cellspacing="0" width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
            <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>告警名称</strong> 
              </td>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>告警级别</strong> 
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>实例</strong> 
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>所属系统</strong> 
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>厂商</strong> 
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>触发时间</strong>
              </td>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>说明</strong> 
              </td>
            </tr>
            {{ range .Alerts.Firing }}
            <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
               <!-- {{ .Labels.alertname }} -->
                {{ .Labels.severity }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Status }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Labels.instance }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Labels.ownningsystem }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Labels.company }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{  .StartsAt.Format "2006-01-02 15:04:05"  }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Annotations.description  }}
              </td>
            </tr>
            {{ end }}
          </table>
        </td>
      </tr>
      </table>
    </table>

    <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;">
      <table width="100%" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
        <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
          <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; text-align: center; color: #999; margin: 0; padding: 0 0 20px;" align="center" valign="top"><a href="{{ .ExternalURL }}" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 12px; color: #999; text-decoration: underline; margin: 0;">Sent by {{ template "__alertmanager" . }}</a></td>
        </tr>
      </table>
    </div></div>
</table>

</body>
</html>
{{ end }}

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
#!/bin/bash
echo "Asia/Shanghai" > /etc/timezone

port=9093

docker stop alertmanager
docker rm alertmanager
docker run --name alertmanager -d \
--net host \
-v /etc/timezone:/etc/timezone:ro \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro \
-v $(pwd)/config/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
-v $(pwd)/templates/:/etc/alertmanager/templates/ \
prom/alertmanager:v0.26.0

配置增加

1
2
3
4
5
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - '10.0.18.2:9093'