prometheus监控kubernetes集群配置详解(七)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

apiVersion: v1
kind: Endpoints
metadata:
  name: ep-test
subsets:
  - addresses:
    - ip: 10.244.3.18
      nodeName: test-k8s-node3
      targetRef:
        kind: Pod
        name: mysql-hcrr6
        namespace: default
    ports:
    - name: mysql
      port: 3306

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

# 抓取指标配置
# 抓取配置
scrape_configs:
  # 任务名称
  - job_name: node_exporter
    kubernetes_sd_configs:
    # 通过node角色发现目标ip端口
    - role: node
    # 重写标签
    relabel_configs:
    # 将目标地址的端口改为node_exporter的端口
    - source_labels: [__address__]
      regex: "(.*):10250"
      target_label: __address__
      replacement: "${1}:9100"
    # 将正则匹配的标签名称，替换原标签名称.(replacement默认是$1),下面的内容就相当于将`__meta_kubernetes_node_label_xxxx`替换为xxxx
    - action: labelmap
      regex: "__meta_kubernetes_node_label_(.*)"

  # 任务名称，这里抓取的是kubelet的指标(默认10250端口)，可以通过curl获取: curl -k 'https://localhost:10250/metrics' --header 'Authorization: Bearer xxxxx'
  - job_name: 'kubernetes-kubelet'
    # 指定通过https还是http抓取指标，默认http
    scheme: https
    tls_config:
      # 用于验证API服务器证书的CA证书。
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      # 如果证书是自签名的,则需要跳过校验
      insecure_skip_verify: true
    # 访问metric api的token，这里对应的就是k8s的serviceaccount，权限可自行配置
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    kubernetes_sd_configs:
    - role: node
    relabel_configs:
    - action: labelmap
      regex: __meta_kubernetes_node_label_(.+)


 # 这里抓取的是cadvisor指标，提供容器相关的指标
  - job_name: 'kubernetes-cadvisor'
    kubernetes_sd_configs:
    - role: node
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecure_skip_verify: true
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
    - action: labelmap
      regex: __meta_kubernetes_node_label_(.+)
      # 修改__metrics_path__标签，会改变metrics的抓取路径。也可以使用kubelet-ip:10250/metrics/cadvisor来抓取
    - action: replace
      source_labels: [__meta_kubernetes_node_name]
      regex: (.*)
      target_label: __metrics_path__
      replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
    - action: replace
      source_labels: [__address__]
      target_label: __address__
      replacement: kubernetes.default.svc:443
    metric_relabel_configs:
    - regex: kernelVersion
      action: labeldrop

  # 监控api-server
  - job_name: 'kubernetes-api-services'
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    kubernetes_sd_configs:
    - role: endpoints
    relabel_configs:
    - action: replace
      source_labels: [__address__]
      target_label: __address__
      replacement: kubernetes.default:443
    - action: keep
      source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_endpoint_port_name,__meta_kubernetes_service_name]
      regex: default;https;kubernetes
    - action: labelmap
      regex: __meta_kubernetes_(.+)

  # 获取kube-state-metrics提供的指标
  - job_name: 'kubernetes-service-endpoints'
    kubernetes_sd_configs:
    - role: endpoints
    relabel_configs:
    # 保留__meta_kubernetes_service_annotation_prometheus_io_scrape=true的标签
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    # 将__address__的端口替换为注解中指定的端口
    - action: replace
      source_labels: [__address__,__meta_kubernetes_service_annotation_prometheus_io_port]
      target_label: __address__
      regex: (.*?):(\d+);(\d+)
      replacement: ${1}:${3}
    # 获取scheme,并将匹配的结果写入__scheme__标签中
    - action: replace
      source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
      target_label: __scheme__
      regex: (https?)
    # 将__metrics_path__的结果替换为注解中的metrics指定的路径
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - action: labelmap
      regex: __meta_kubernetes_service_label_(.+)
    - source_labels: [__meta_kubernetes_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_service_name]
      action: replace
      target_label: kubernetes_name

  # 抓取nginx-ingress的metrics
  - job_name: 'ingress-nginx'
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    kubernetes_sd_configs:
    - role: service
      namespaces:
        names:
          - ingress-nginx
    relabel_configs:
      - source_labels: [__address__]
        regex: (.*)10254
        action: keep