cadvisor容器监控

# 1.安装部署

被监控机器上找个目录放置,这里我们以 `/opt/agent` 为例

- vim /opt/agent/docker-compose.yml

```yaml
version: '3'

services:
    cadvisor:
        image: google/cadvisor:latest
        container_name: cadvisor
        hostname: cadvisor
        restart: always
        volumes:
            - /:/rootfs:ro
            - /var/run:/var/run:rw
            - /sys:/sys:ro
            - /var/lib/docker/:/var/lib/docker:ro
        ports:
            - "8080:8080"
```

# 2.启动

`docker-compose up -d`

# 3.配置自动发现的主机列表

基于 file_sd_configs

- yaml格式

```yaml
# vim /opt/jonnyan404/prometheus/target/docker.yml
- targets: ['192.168.1.220:8080']
  labels:
    env:   'docker'
    hostname: 'one-docker'

- targets: ['192.168.1.221:8080']
  labels:
    env:   'docker'
    hostname: 'two-docker'
```

# 4. 配置告警规则

- vim /opt/jonnyan404/prometheus/rules/docker-exporter-record.yml

```yaml
groups:
  - name: dockersInfo-record
    rules:
    - expr: count by (instance, name) (count_over_time(container_last_seen{job="aws_ec2", name!="", container_label_restartcount!=""}[15m]))
      record: dockersInfo:container:restart
      labels:
        desc: "15m周期内容器发生重启的次数"
        unit: ""
        job: "aws_ec2"
##############################################################################################
#                              cpu                                                           #
    - expr: sum by(name) (irate(container_cpu_usage_seconds_total{job="aws_ec2", name!=''}[5m]))  * 100
      record: dockersInfo:container:cpu:total:percent
      labels:
        desc: "容器的cpu总消耗百分比"
        unit: "%"
        job: "aws_ec2"

- expr: irate(container_fs_io_time_seconds_total{job="aws_ec2", name!=''}[5m])  * 100
      record: dockersInfo:cpu:iowait:percent
      labels:
        desc: "容器的cpu iowait百分比"
        unit: "%"
        job: "aws_ec2"

##############################################################################################
#                                    memory                                                  #
#    - expr: container_spec_memory_limit_bytes{job="aws_ec2", name!=''}
#      record: dockersInfo:memory:total
#      labels:
#        desc: "容器的内存总量"
#        unit: byte
#        job: "aws_ec2"

# - expr: container_memory_usage_bytes{job="aws_ec2", name!=''} / container_spec_memory_limit_bytes{job="aws_ec2", name!=''} * 100
    #   record: dockersInfo:memory:used:percent
    #   labels:
    #     desc: "容器的内存使用百分比"
    #     unit: "%"
    #     job: "aws_ec2"

- expr: (container_memory_usage_bytes{job="aws_ec2", name!=''} - container_memory_cache{job="aws_ec2", name!=''}) / 1024 / 1024 / 1024
      record: dockersInfo:memory:used
      labels:
        desc: "容器已使用内存"
        unit: "G"
        job: "aws_ec2"
```

- vim /opt/jonnyan404/prometheus/rules/node-exporter-alert.yml

```yaml
groups:
  - name: container-alert
    rules:
    - alert: container-restart-times-high
      expr: dockersInfo:container:restart > 5
      for: 1m
      labels:
        severity: warn
      annotations:
        summary: "instance: {{ $labels.instance }} 下的 container: {{ $labels.name }} 15分钟内重启次数超过5次"
        grafana: "http://x.x.x.x:3000/d/9eKutrCMk/docker-monitoring?orgId=1&refresh=10s"

- alert: container-cpu-usage-high
      expr: dockersInfo:container:cpu:total:percent > 400
      for: 5m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 下的 container: {{ $labels.name }} cpu 使用率持续超过4个核心."
        grafana: "http://x.x.x.x:3000/d/9eKutrCMk/docker-monitoring?orgId=1&refresh=10s"

- alert: container-cpu-iowait-high
      expr: dockersInfo:cpu:iowait:percent > 10
      for: 1m
      labels:
        severity: warn
      annotations:
        summary: "instance: {{ $labels.instance }} 下的 container: {{ $labels.name }} cpu iowait 持续超过10%"
        grafana: "http://x.x.x.x:3000/d/9eKutrCMk/docker-monitoring?orgId=1&refresh=10s"

- alert: 容器内存占用异常
      expr: dockersInfo:memory:used > 1.5
      for: 1m
      labels:
        severity: warn
      annotations:
        summary: "instance: {{ $labels.instance }} 下的 container: {{ $labels.name }} 已使用内存持续超过1.5G"
        grafana: "http://x.x.x.x:3000/d/9eKutrCMk/docker-monitoring?orgId=1&refresh=10s"
```

# 5.重启prometheus,使规则生效

```
docker restart prometheus
```

# 6.导入grafana模板

- ID: 193