部署Blackbox Exporter

Download file: https://shackles.cn/Introduction/blackbox-deploymeny.yaml

1、创建YAML配置文件（blackbox-deploymeny.yaml）

root@master:~#cat blackbox-deploymeny.yaml
apiVersion: v1
kind: Service
metadata:
  name: blackbox
  namespace: kubesphere-monitoring-system
  labels:
    app: blackbox
spec:
  selector:
    app: blackbox
  ports:
    - port: 9115
      targetPort: 9115
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: blackbox-config
  namespace: kubesphere-monitoring-system
data:
  blackbox.yaml: |-
    modules:
      http_2xx:
        prober: http
        timeout: 10s
        http:
          valid_http_versions: ["HTTP/1.1", "HTTP/2"]
          valid_status_codes: []
          method: GET
          preferred_ip_protocol: "ip4"
      http_post_2xx:
        prober: http
        timeout: 10s
        http:
          valid_http_versions: ["HTTP/1.1", "HTTP/2"]
          valid_status_codes: []
          method: POST
          preferred_ip_protocol: "ip4"
      tcp_connect:
        prober: tcp
        timeout: 10s
      pop3s_banner:
        prober: tcp
        tcp:
          query_response:
          - expect: "^+OK"
          tls: true
          tls_config:
            insecure_skip_verify: false
      grpc:
        prober: grpc
        grpc:
          tls: true
        preferred_ip_protocol: "ip4"
      grpc_plain:
        prober: grpc
        grpc:
          tls: false
          service: "service1"
      ssh_banner:
        prober: tcp
        tcp:
          query_response:
          - expect: "^SSH-2.0-"
          - send: "SSH-2.0-blackbox-ssh-check"
      irc_banner:
        prober: tcp
        tcp:
          query_response:
          - send: "NICK prober"
          - send: "USER prober prober prober :prober"
          - expect: "PING :([^ ]+)"
            send: "PONG ${1}"
          - expect: "^:[^ ]+ 001"
      icmp:
        prober: icmp
      icmp_ttl5:
        prober: icmp
        timeout: 5s
        icmp:
          ttl: 5
      ping:
        prober: icmp
        timeout: 5s
        icmp:
          preferred_ip_protocol: "ip4"
      dns:
        prober: dns
        dns:
          transport_protocol: "tcp"
          preferred_ip_protocol: "ip4"
          query_name: "kubernetes.defalut.svc.cluster.local"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: blackbox
  namespace: kubesphere-monitoring-system
spec:
  selector:
    matchLabels:
      app: blackbox
  template:
    metadata:
      labels:
        app: blackbox
    spec:
      containers:
        - name: blackbox
          image: prom/blackbox-exporter:v0.23.0
          args:
            - "--config.file=/etc/blackbox_exporter/blackbox.yaml"
            - "--log.level=error"
          ports:
            - containerPort: 9115
          volumeMounts:
            - name: config
              mountPath: /etc/blackbox_exporter
      volumes:
        - name: config
          configMap:
            name: blackbox-config

2、创建blackbox_exporter Pod

kubectl apply -f blackbox-deploymeny.yaml

配置监控

由于KubeSphere集群是用的Prometheus Operator方式部署的，所以就以additional的形式添加配置。以下配置适用于kube-prometheus 0.7.0及以下版本

1、创建prometheus-additional.yaml文件，定义内容如下：

Download file:https://shackles.cn/Introduction/prometheus-additional.yaml

root@master:~#cat prometheus-additional.yaml
- job_name: "test-service-status"
  metrics_path: /probe
  params:
    module: [http_2xx]  # Look for a HTTP 200 response.
  static_configs:
    - targets:
        - https://prometheus.io/
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      replacement: blackbox.kubesphere-monitoring-system:9115

2、创建secret

kubectl -n kubesphere-monitoring-system create secret generic additional-config --from-file=prometheus-additional.yaml

3、修改prometheus的配置,添加以下三行配置信息

root@master:~#kubectl edit prometheus -n kubesphere-monitoring-system k8s
..............
spec:
  additionalScrapeConfigs:
    key: prometheus-additional.yaml
    name: additional-config
..............

4、reload prometheus

4.1、查看Prometheus的SVCIP

root@master:~#kubectl get svc -n kubesphere-monitoring-system -l app.kubernetes.io/instance=k8s
NAME             TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)                         AGE
prometheus-k8s   NodePort   10.233.23.88   <none>        9090:32205/TCP,8080:32009/TCP   14d

4.2、执行热加载reload

curl -X POST "http://10.233.23.88:9090/-/reload"

等待一段时间，在prometheus的web界面中点击status选择Targets，即可看到上方定义的test-service-status
后面修改配置文件，使用以下三条命令即可：

kubectl delete secret additional-config -n kubesphere-monitoring-system
kubectl -n kubesphere-monitoring-system create secret generic additional-config --from-file=prometheus-additional.yaml
curl -X POST "http://10.233.23.88:9090/-/reload"