{"id":4268,"date":"2021-11-26T15:24:03","date_gmt":"2021-11-26T23:24:03","guid":{"rendered":"https:\/\/www.xh86.me\/?p=4268"},"modified":"2021-11-26T15:25:28","modified_gmt":"2021-11-26T23:25:28","slug":"prometheus%e7%9b%91%e6%8e%a7%e7%b3%bb%e5%88%97%e4%ba%8c-docker%e5%ae%b9%e5%99%a8%e5%8c%96%e9%83%a8%e7%bd%b2%e5%ae%9e%e6%88%98","status":"publish","type":"post","link":"https:\/\/www.xh86.me\/?p=4268","title":{"rendered":"Prometheus\u76d1\u63a7\u7cfb\u5217\u4e8c | Docker\u5bb9\u5668\u5316\u90e8\u7f72\u5b9e\u6218"},"content":{"rendered":"

1 \u90e8\u7f72 Docker \u670d\u52a1<\/h2>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
curl https:\/\/mirrors.aliyun.com\/docker-ce\/linux\/centos\/docker-ce.repo -o \/etc\/yum.repos.d\/docker.repo<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
yum list docker-ce –showduplicates | sort -r # \u663e\u793a\u6240\u6709\u7248\u672c<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
yum install -y docker-ce-20.10.5 # \u6307\u5b9a docker \u7248\u672c<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
systemctl start docker # \u542f\u52a8 docker \u670d\u52a1<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
systemctl status docker # \u67e5\u770b docker \u670d\u52a1\u72b6\u6001<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
systemctl enable<\/span> docker # \u8bbe\u7f6e docker \u5f00\u673a\u81ea\u542f\u52a8<\/span><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
2 \u90e8\u7f72 Prometheus \u670d\u52a1<\/h2>\n
\u521b\u5efa mon \u7528\u6237\uff0c\u521b\u5efa\u76ee\u5f55\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
groupadd -g 2000 mon<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
useradd -u 2000 -g mon mon<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
mkdir -p \/home\/mon\/prometheus\/{etc,data,rules}<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u521b\u5efa\u914d\u7f6e\u6587\u4ef6\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
vim \/home\/mon\/prometheus\/etc\/prometheus.yml<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
# my global config<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
global:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# scrape_timeout is set to the global default (10s).<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# Alertmanager configuration<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
alerting:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
alertmanagers:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– static_configs:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– targets:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# – alertmanager:9093<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# Load rules once and periodically evaluate them according to the global ‘evaluation_interval’.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
rule_files:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# – “first_rules.yml”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# – “second_rules.yml”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# A scrape configuration containing exactly one endpoint to scrape:<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# Here it’s Prometheus itself.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
scrape_configs:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– job_name: ‘prometheus’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# metrics_path defaults to ‘\/metrics’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# scheme defaults to ‘http’.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
static_configs:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– targets: [‘localhost:9090’<\/span>]<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u542f\u52a8\u5bb9\u5668\u670d\u52a1\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
docker pull prom\/prometheus<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
cd<\/span> \/home\/mon\/<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
chown mon. -R prometheus<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
docker run -d –user root -p 9090:9090 –name prometheus \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/prometheus\/etc\/prometheus.yml:\/etc\/prometheus\/prometheus.yml \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/prometheus\/rules:\/etc\/prometheus\/rules \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/prometheus\/data:\/data\/prometheus \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
prom\/prometheus \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–config.file=“\/etc\/prometheus\/prometheus.yml”<\/span> \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–storage.tsdb.path=“\/data\/prometheus”<\/span> \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–web.listen-address=“0.0.0.0:9090”<\/span><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
3 \u90e8\u7f72 Grafana \u670d\u52a1<\/h2>\n
\u521b\u5efa\u6570\u636e\u76ee\u5f55\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
mkdir -p \/home\/mon\/grafana\/plugins<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u5b89\u88c5\u63d2\u4ef6\uff1a\u4e0b\u8f7dGrafana\u63d2\u4ef6<\/a><\/p>\n
<\/code><\/pre>\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
tar zxf \/tmp\/grafana-plugins.tar.gz -C \/home\/mon\/grafana\/plugins\/<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
chown -R mon. \/home\/mon\/grafana<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
chmod 777 -R \/home\/mon\/grafana<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u542f\u52a8\u5bb9\u5668\u670d\u52a1\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n
\n
<\/div>\n<\/td>\n
docker pull grafana\/grafana:latest<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
docker run -d -p 3000:3000 -v \/home\/mon\/grafana:\/var\/lib\/grafana –name=grafana grafana\/grafana:latest<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
4 \u914d\u7f6e Grafana \u5bf9\u63a5 Prometheus<\/h2>\n
\u8bbf\u95ee\u00a0http:\/\/ip:3000<\/code>\uff0c\u521d\u59cb\u8d26\u53f7\u5bc6\u7801\u4e3a\u00a0admin\/admin<\/code>\uff0c\u4f1a\u8981\u6c42\u66f4\u6539\u5bc6\u7801\u3002<\/p>\n
\u6309\u7167\u5982\u4e0b\u622a\u56fe\u987a\u5e8f\u914d\u7f6e Prometheus Dashboard\uff1a<\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
5 \u90e8\u7f72 Node_Exporter \u670d\u52a1<\/h2>\n
\u6211\u4ee5\u76d1\u63a7\u4e00\u53f0\u963f\u91cc\u4e91ECS\u4e3a\u4f8b\u3002<\/p><\/blockquote>\n
\u5b89\u88c5\u914d\u7f6eNode_Exporter\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
curl https:\/\/github.com\/prometheus\/node_exporter\/releases\/download\/v1.1.1\/node_exporter-1.1.1.linux-amd64.tar.gz > \/opt\/node_exporter-1.1.1.linux-amd64.tar.gz<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
cd<\/span> \/opt<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
tar zxf node_exporter-1.1.1.linux-amd64.tar.gz<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
mv node_exporter-1.1.1.linux-amd64 node_exporter<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u914d\u7f6e\u670d\u52a1\u542f\u52a8\u811a\u672c\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
vim \/usr\/lib\/systemd\/system\/node_exporter.service<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
[Unit]<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
Description=node_exporter service<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
[Service]<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
User=root<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
ExecStart=\/opt\/node_exporter\/node_exporter<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
TimeoutStopSec=10<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
Restart=on-failure<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
RestartSec=5<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
[Install]<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
WantedBy=multi-user.target<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
<\/code><\/pre>\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
systemctl daemon-reload<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
systemctl start node_exporter<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
systemctl status node_exporter<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
systemctl enable<\/span> node_exporter<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u5728\u8fd9\u53f0ECS\u670d\u52a1\u5668\u7684Nginx\u4e0b\u914d\u7f6e\u53cd\u5411\u4ee3\u7406\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
vim \/opt\/nginx\/conf\/conf.d\/blog.conf<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# prometheus monitor node exporter<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
location \/node\/service {<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
proxy_pass http:\/\/127.0.0.1:9100\/metrics;<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
proxy_set_header Host $http_host<\/span>;<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
proxy_set_header X-Real-IP $remote_addr<\/span>;<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for<\/span>;<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
proxy_set_header X-Forwarded-Proto $scheme<\/span>;<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
}<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
\/opt\/nginx\/sbin\/nginx -s reload<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u5728Prometheus\u670d\u52a1\u5668\u7aef\u4fee\u6539 \u914d\u7f6e\u6587\u4ef6\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
vim \/home\/mon\/prometheus\/etc\/prometheus.yml<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– job_name: ‘node-service’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
static_configs:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– targets: [‘blog.iuskye.com’<\/span>]<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: node-service<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
scheme: https<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
metrics_path: \/node\/service<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u91cd\u542f Prometheus \u5bb9\u5668\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
docker restart prometheus<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u9a8c\u8bc1\u662f\u5426\u83b7\u53d6\u5230\u6570\u636e\uff0c\u5728\u6d4f\u89c8\u5668\u8f93\u5165\uff1ahttps:\/\/blog.iuskye.com\/node\/service<\/code>:<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# TYPE go_gc_duration_seconds summary<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_gc_duration_seconds{quantile=“0”<\/span>} 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_gc_duration_seconds{quantile=“0.25”<\/span>} 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_gc_duration_seconds{quantile=“0.5”<\/span>} 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_gc_duration_seconds{quantile=“0.75”<\/span>} 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_gc_duration_seconds{quantile=“1”<\/span>} 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_gc_duration_seconds_sum 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_gc_duration_seconds_count 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# HELP go_goroutines Number of goroutines that currently exist.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# TYPE go_goroutines gauge<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_goroutines 7<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# HELP go_info Information about the Go environment.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# TYPE go_info gauge<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
go_info{version=“go1.15.8”<\/span>} 1<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# TYPE go_memstats_alloc_bytes gauge<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u521b\u5efaDashboard\uff1a<\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
\"\"<\/div><\/a><\/p>\n
6 \u90e8\u7f72 Alertmanager \u670d\u52a1<\/h2>\n
\u521b\u5efa\u76ee\u5f55\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n
\n
<\/div>\n<\/td>\n
mkdir -p \/home\/mon\/alertmanager\/{etc,data}<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
chmod 777 -R \/home\/mon\/alertmanager<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u521b\u5efa\u914d\u7f6e\u6587\u4ef6\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
vim \/home\/mon\/alertmanager\/etc\/alertmanager.yml<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
global:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
resolve_timeout: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
smtp_from: ‘319981932@qq.com’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
smtp_smarthost: ‘smtp.qq.com:465’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
smtp_auth_username: ‘319981932@qq.com’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# \u6ce8\u610f\u8fd9\u91cc\u9700\u8981\u914d\u7f6eQQ\u90ae\u7bb1\u7684\u6388\u6743\u7801\uff0c\u4e0d\u662f\u767b\u5f55\u5bc6\u7801\uff0c\u6388\u6743\u7801\u5728\u8d26\u6237\u914d\u7f6e\u4e2d\u67e5\u770b<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
smtp_auth_password: ‘abcdefghijklmmop’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
smtp_require_tls: false<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
route:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
group_by: [‘alert_node’<\/span>]<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
group_wait: 5s<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
group_interval: 5s<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
repeat_interval: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
receiver: ’email’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
receivers:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– name: ’email’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
email_configs:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# \u8bf7\u6ce8\u610f\u8fd9\u91cc\u7684\u6536\u4ef6\u7bb1\u8bf7\u6539\u4e3a\u4f60\u81ea\u5df1\u7684\u90ae\u7bb1\u5730\u5740<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– to: ‘319981932@qq.com’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
send_resolved: true<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
inhibit_rules:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– source_match:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: ‘critical’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
target_match:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: ‘warning’<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
equal: [‘alert_node’<\/span>, ‘dev’<\/span>, ‘instance’<\/span>]<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u62c9\u53d6\u955c\u50cf\u5e76\u542f\u52a8\u5bb9\u5668\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
docker pull prom\/alertmanager:latest<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
chown -R mon. alertmanager\/<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
docker run -d –user root -p 9093:9093 –name alertmanager \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/alertmanager\/etc\/alertmanager.yml:\/etc\/alertmanager\/alertmanager.yml \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/alertmanager\/data:\/alertmanager\/data<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
prom\/alertmanager:latest \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–config.file=“\/etc\/alertmanager\/alertmanager.yml”<\/span> \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–web.listen-address=“0.0.0.0:9093”<\/span><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u67e5\u770balertmanager\u5bb9\u5668IP\u5730\u5740\uff0c\u7528\u4e8e\u914d\u7f6eprometheus\u5bf9\u63a5\u63a5\u53e3\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
docker exec<\/span> -it alertmanager \/bin\/sh -c “ip a”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
link\/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
inet 127.0.0.1\/8 scope host lo<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
valid_lft forever preferred_lft forever<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
10: eth0@if11: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
link\/ether 02:42:ac:11:00:04 brd ff:ff:ff:ff:ff:ff<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
inet 172.17.0.4\/16 brd 172.17.255.255 scope global eth0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
valid_lft forever preferred_lft forever<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u4fee\u6539prometheus\u914d\u7f6e\u6587\u4ef6\u5bf9\u63a5alertmanager\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
vim \/home\/mon\/prometheus\/etc\/prometheus.yml<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
alerting:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
alertmanagers:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– static_configs:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– targets:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– 172.17.0.4:9093<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
rule_files:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– “\/etc\/prometheus\/rules\/*rules.yml”<\/span><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u914d\u7f6e\u544a\u8b66\u89c4\u5219\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
vim \/home\/mon\/prometheus\/rules\/alert-node-rules.yml<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
groups:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– name: alert-node<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
rules:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeDown<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
# \u6ce8\u610f\uff1a\u8fd9\u91cc\u7684job_name\u4e00\u5b9a\u8981\u8ddfprometheus\u914d\u7f6e\u6587\u4ef6\u4e2d\u914d\u7f6e\u7684\u76f8\u5339\u914d<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: up{job=“node-service”<\/span>} == 0<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 1m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: critical<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} is down”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Instance: {{ $labels<\/span>.instance }} \u5df2\u7ecf\u5b95\u673a 1\u5206\u949f”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeCpuHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: (1 – avg by (instance) (irate(node_cpu_seconds_total{job=“node-service”<\/span>,mode=“idle”<\/span>}[5m]))) * 100 > 80<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} cpu\u4f7f\u7528\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “CPU \u4f7f\u7528\u7387\u8d85\u8fc7 80%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeCpuIowaitHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: avg by (instance) (irate(node_cpu_seconds_total{job=“node-service”<\/span>,mode=“iowait”<\/span>}[5m])) * 100 > 50<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} cpu iowait \u4f7f\u7528\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “CPU iowait \u4f7f\u7528\u7387\u8d85\u8fc7 50%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeLoad5High<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job=“node-service”<\/span>,mode=‘system’<\/span>})) * 1.2<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} load(5m) \u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Load(5m) \u8fc7\u9ad8\uff0c\u8d85\u51facpu\u6838\u6570 1.2\u500d”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeMemoryHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: (1 – node_memory_MemAvailable_bytes{job=“node-service”<\/span>} \/ node_memory_MemTotal_bytes{job=“node-service”<\/span>}) * 100 > 60<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} memory \u4f7f\u7528\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Memory \u4f7f\u7528\u7387\u8d85\u8fc7 90%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeDiskRootHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: (1 – node_filesystem_avail_bytes{job=“node-service”<\/span>,fstype=~“ext.*|xfs”<\/span>,mountpoint =“\/”<\/span>} \/ node_filesystem_size_bytes{job=“node-service”<\/span>,fstype=~“ext.*|xfs”<\/span>,mountpoint =“\/”<\/span>}) * 100 > 90<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 10m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk(\/ \u5206\u533a) \u4f7f\u7528\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk(\/ \u5206\u533a) \u4f7f\u7528\u7387\u8d85\u8fc7 90%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeDiskBootHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: (1 – node_filesystem_avail_bytes{job=“node-service”<\/span>,fstype=~“ext.*|xfs”<\/span>,mountpoint =“\/boot”<\/span>} \/ node_filesystem_size_bytes{job=“node-service”<\/span>,fstype=~“ext.*|xfs”<\/span>,mountpoint =“\/boot”<\/span>}) * 100 > 80<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 10m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk(\/boot \u5206\u533a) \u4f7f\u7528\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk(\/boot \u5206\u533a) \u4f7f\u7528\u7387\u8d85\u8fc7 80%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeDiskReadHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: irate(node_disk_read_bytes_total{job=“node-service”<\/span>}[5m]) > 20 * (1024 ^ 2)<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk \u8bfb\u53d6\u5b57\u8282\u6570 \u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk \u8bfb\u53d6\u5b57\u8282\u6570 \u901f\u7387\u8d85\u8fc7 20 MB\/s”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeDiskWriteHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: irate(node_disk_written_bytes_total{job=“node-service”<\/span>}[5m]) > 20 * (1024 ^ 2)<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk \u5199\u5165\u5b57\u8282\u6570 \u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk \u5199\u5165\u5b57\u8282\u6570 \u901f\u7387\u8d85\u8fc7 20 MB\/s”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeDiskReadRateCountHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: irate(node_disk_reads_completed_total{job=“node-service”<\/span>}[5m]) > 3000<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk iops \u6bcf\u79d2\u8bfb\u53d6\u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk iops \u6bcf\u79d2\u8bfb\u53d6\u901f\u7387\u8d85\u8fc7 3000 iops”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeDiskWriteRateCountHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: irate(node_disk_writes_completed_total{job=“node-service”<\/span>}[5m]) > 3000<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 5m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk iops \u6bcf\u79d2\u5199\u5165\u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk iops \u6bcf\u79d2\u5199\u5165\u901f\u7387\u8d85\u8fc7 3000 iops”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeInodeRootUsedPercentHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: (1 – node_filesystem_files_free{job=“node-service”<\/span>,fstype=~“ext4|xfs”<\/span>,mountpoint=“\/”<\/span>} \/ node_filesystem_files{job=“node-service”<\/span>,fstype=~“ext4|xfs”<\/span>,mountpoint=“\/”<\/span>}) * 100 > 80<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 10m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk(\/ \u5206\u533a) inode \u4f7f\u7528\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk (\/ \u5206\u533a) inode \u4f7f\u7528\u7387\u8d85\u8fc7 80%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeInodeBootUsedPercentHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: (1 – node_filesystem_files_free{job=“node-service”<\/span>,fstype=~“ext4|xfs”<\/span>,mountpoint=“\/boot”<\/span>} \/ node_filesystem_files{job=“node-service”<\/span>,fstype=~“ext4|xfs”<\/span>,mountpoint=“\/boot”<\/span>}) * 100 > 80<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 10m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} disk(\/boot \u5206\u533a) inode \u4f7f\u7528\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Disk (\/boot \u5206\u533a) inode \u4f7f\u7528\u7387\u8d85\u8fc7 80%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeFilefdAllocatedPercentHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: node_filefd_allocated{job=“node-service”<\/span>} \/ node_filefd_maximum{job=“node-service”<\/span>} * 100 > 80<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 10m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} filefd \u6253\u5f00\u767e\u5206\u6bd4\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Filefd \u6253\u5f00\u767e\u5206\u6bd4 \u8d85\u8fc7 80%”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeNetworkNetinBitRateHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: avg by (instance) (irate(node_network_receive_bytes_total{device=~“eth0|eth1|ens33|ens37”<\/span>}[1m]) * 8) > 20 * (1024 ^ 2) * 8<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 3m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} network \u63a5\u6536\u6bd4\u7279\u6570 \u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Network \u63a5\u6536\u6bd4\u7279\u6570 \u901f\u7387\u8d85\u8fc7 20MB\/s”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeNetworkNetoutBitRateHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~“eth0|eth1|ens33|ens37”<\/span>}[1m]) * 8) > 20 * (1024 ^ 2) * 8<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 3m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} network \u53d1\u9001\u6bd4\u7279\u6570 \u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Network \u53d1\u9001\u6bd4\u7279\u6570 \u901f\u7387\u8d85\u8fc7 20MB\/s”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeNetworkNetinPacketErrorRateHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: avg by (instance) (irate(node_network_receive_errs_total{device=~“eth0|eth1|ens33|ens37”<\/span>}[1m])) > 15<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 3m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} \u63a5\u6536\u9519\u8bef\u5305 \u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Network \u63a5\u6536\u9519\u8bef\u5305 \u901f\u7387\u8d85\u8fc7 15\u4e2a\/\u79d2”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeNetworkNetoutPacketErrorRateHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: avg by (instance) (irate(node_network_transmit_packets_total{device=~“eth0|eth1|ens33|ens37”<\/span>}[1m])) > 15<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 3m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} \u53d1\u9001\u9519\u8bef\u5305 \u901f\u7387\u8fc7\u9ad8”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Network \u53d1\u9001\u9519\u8bef\u5305 \u901f\u7387\u8d85\u8fc7 15\u4e2a\/\u79d2”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeProcessBlockedHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: node_procs_blocked{job=“node-service”<\/span>} > 10<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 10m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: warning<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} \u5f53\u524d\u88ab\u963b\u585e\u7684\u4efb\u52a1\u7684\u6570\u91cf\u8fc7\u591a”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Process \u5f53\u524d\u88ab\u963b\u585e\u7684\u4efb\u52a1\u7684\u6570\u91cf\u8d85\u8fc7 10\u4e2a”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
– alert: NodeTimeOffsetHigh<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
expr: abs(node_timex_offset_seconds{job=“node-service”<\/span>}) > 3 * 60<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
for<\/span>: 2m<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
labels:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
severity: info<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
instance: “{{ $labels<\/span>.instance }}”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
annotations:<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
summary: “instance: {{ $labels<\/span>.instance }} \u65f6\u95f4\u504f\u5dee\u8fc7\u5927”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
description: “Time \u8282\u70b9\u7684\u65f6\u95f4\u504f\u5dee\u8d85\u8fc7 3m”<\/span><\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
value: “{{ $value<\/span> }}”<\/span><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u91cd\u542fprometheus\u5bb9\u5668\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
docker restart prometheus<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u9a8c\u8bc1\u662f\u5426\u6536\u5230\u544a\u8b66\u90ae\u4ef6\uff0c\u6211\u4eec\u5c06node_exporter\u5173\u95ed\uff0c\u5728\u88ab\u76d1\u63a7\u7684ECS\u670d\u52a1\u5668\u64cd\u4f5c\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
systemctl stop node_exporter<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u7136\u540e\u5237\u65b0prometheus\u7684\u9875\u9762\uff0c\u67e5\u770bAlerts\u83dc\u5355\uff0c\u6211\u4eec\u53d1\u73b0NodeDown\u89c4\u5219\u5904\u4e8ePENDING\u72b6\u6001\uff1a<\/p>\n
\"\"<\/div><\/a><\/p>\n
\u7b49\u5f85\u4e00\u5206\u949f\u540e\u518d\u6b21\u5237\u65b0\u67e5\u770b\uff0c\u5df2\u7ecf\u53d8\u6210\u4e86FIRING\u72b6\u6001\uff1a<\/p>\n
\"\"<\/div><\/a><\/p>\n
\u8fd9\u65f6\u5019\u6211\u4eec\u53bb\u67e5\u770b\u4e0b\u90ae\u7bb1\uff1a<\/p>\n
\"\"<\/div><\/a><\/p>\n
\u8bf4\u660e\u5df2\u7ecf\u6536\u5230\u4e86\u544a\u8b66\u90ae\u4ef6\u3002\u73b0\u5728\u6211\u4eec\u628a\u5b83\u6062\u590d\uff1a<\/p>\n
<\/code><\/pre>\n\n\n\n
\n
<\/div>\n<\/td>\n
systemctl start node_exporter<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u7136\u540e\u6211\u4eec\u5c31\u6536\u5230\u4e86\u670d\u52a1\u6062\u590d\u7684\u544a\u8b66\u90ae\u4ef6\uff1a<\/p>\n
\"\"<\/div><\/a><\/p>\n
7 \u591a\u5bb9\u5668\u542f\u52a8\u7ba1\u7406<\/h2>\n
\u591a\u5bb9\u5668\u914d\u7f6e\uff0c\u9700\u8981\u4fee\u6539\u7aef\u53e3\u3001\u6570\u636e\u5b58\u50a8\u8def\u5f84\u7b49\u4fe1\u606f\uff0c\u4f8b\u5982\uff1a<\/p>\n
Prometheus<\/p><\/blockquote>\n
<\/code><\/pre>\n\n\n\n\n\n\n\n\n\n\n
\n
<\/div>\n<\/td>\n
docker run -d –user root -p 9091:9090 –name prometheus-poc \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/prometheus-poc\/etc\/prometheus.yml:\/etc\/prometheus\/prometheus.yml \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/prometheus-poc\/rules:\/etc\/prometheus\/rules \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
-v \/home\/mon\/prometheus-poc\/data:\/data\/prometheus \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
prom\/prometheus \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–config.file=“\/etc\/prometheus\/prometheus.yml”<\/span> \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–storage.tsdb.path=“\/data\/prometheus”<\/span> \\<\/td>\n<\/tr>\n
\n
<\/div>\n<\/td>\n
–web.listen-address=“0.0.0.0:9090”<\/span><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
<\/code><\/pre>\n
\n
<\/div>\n
<\/div>\n<\/div>\n
\u4e0d\u540c\u4e4b\u5904\uff1a<\/p>\n