Thanos Receiver 对接 Prometheus operator
|
231
|
0
|
Thanos

569 字
|
19 分钟

在很多环境中我们的 Prometheus 可以能是通过 operator 部署在 K8S 集群中，所以我们还需要将 Prometheus 对接到 thanos 中具体操作如下：

注意：

前提是上面的 minio 和 longhorn 部署好的情况下在继续下面的操作：

6.4.1 对接 minio

1 对接 minio

root@master:~/prometheus# cat thanos-storage-minio.yaml 
type: s3
config:
  bucket: thanos
  # 访问 minio 地址，由于这里都是在 K8S 里面使用所以采用了 dns 的方式访问它的 API
  endpoint: minio.default.svc.cluster.local:9000
  access_key: minio
  secret_key: minio123
  insecure: true
  signature_version2: false

2 使用上面的配置文件来创建一个 Secret 对象：

# 部署在 monitoring NS 下
root@master:~/thanos# kubectl create secret generic thanos-objectstorage --from-file=thanos.yaml=thanos-storage-minio.yaml -n monitoring 

# 创建成功
root@master:~/prometheus# kubectl get secrets -n monitoring 
NAME                     TYPE                                  DATA   AGE
thanos-objectstorage     Opaque                                1      16s

6.4.2 创建 store

下面都属部署在 monitoring NS 中

root@master:~/prometheus# cat thanos-store.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: thanos-store-gateway
  namespace: monitoring
  labels:
    app: thanos-store-gateway
spec:
  # 副本数配置为 2 用于高可用
  replicas: 2
  selector:
    matchLabels:
      app: thanos-store-gateway
  # 匹配 headless Service
  serviceName: thanos-store-gateway
  template:
    metadata:
      labels:
        app: thanos-store-gateway
        # thanos-store-api: "true" 该标签就是想告诉系统当前的这个组件也实现了 store-api，然后 query 组件就能直接对接有该标签的组件，所以后续 query 就能够实现直接查询拥有该标签的数据比如 sidecar 和 store 组件
        thanos-store-api: "true"
    spec:
      affinity:
        # 由于是采用了高可用，所以这里我使用了 pod 的反亲和性将两个 pod 部署在不同的 node 上
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                topologyKey: kubernetes.io/hostname
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - thanos-store-gateway
      containers:
        - name: thanos
          image: thanosio/thanos:v0.25.1
          args:
            - "store"
            - "--log.level=debug"
            - "--data-dir=/data"
        # objstore.config-file 指定对象存储的配置文件，将刚才 minio 的对接访问通过 secrets 引用进来
            - "--objstore.config-file=/etc/secret/thanos.yaml"
            - "--index-cache-size=500MB"
            - "--chunk-pool-size=500MB"
          ports:
            - name: http
              containerPort: 10902
            - name: grpc
              containerPort: 10901
          livenessProbe:
            httpGet:
              port: 10902
              path: /-/healthy
          readinessProbe:
            httpGet:
              port: 10902
              path: /-/ready
          volumeMounts:
      # 将 thanos-objectstorage secrete 挂载到 /etc/secret 用于上面 objstore.config-file 指定
            - name: object-storage-config
              mountPath: /etc/secret
              readOnly: false
            - mountPath: /data
              name: data
      volumes:
        # 将 thanos-objectstorage secrete 挂载进容器中
        - name: object-storage-config
          secret:
            secretName: thanos-objectstorage
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes:
          - ReadWriteOnce
        storageClassName: longhorn
        resources:
          requests:
            storage: 1Gi
root@master:~/prometheus# kubectl apply -f thanos-store.yaml 

root@master:~/prometheus# kubectl get pod -n monitoring | grep store
thanos-store-gateway-0                 1/1     Running   0          84m
thanos-store-gateway-1                 1/1     Running   0          83m

6.4.3 部署 receiver

1 创建 receiver-hashring

root@master:~/prometheus# cat thanos-receiver-hashring.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: hashring-config
  namespace: monitoring
data:
  hashring.json: |-
    [
      {
        "endpoints": [ 
            "thanos-receiver-0.thanos-receiver:10901",
            "thanos-receiver-1.thanos-receiver:10901",
            "thanos-receiver-2.thanos-receiver:10901"
        ]
      }
    ]

root@master:~/prometheus# kubectl apply -f thanos-receiver-hashring.yaml

2 创建 receiver

root@master:~/prometheus# cat thanos-receiver.yaml 
# thanos-receiver.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: thanos-receiver
  name: thanos-receiver
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: thanos-receiver
  serviceName: thanos-receiver
  replicas: 3 #节点数量
  template:
    metadata:
      labels:
        app: thanos-receiver
        # 拥有和 query 一样的标签，这样就能将数据传到 query 中并接收 receiver 请求
        thanos-store-api: "true"
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                topologyKey: kubernetes.io/hostname
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - thanos-receiver
      containers:
        - name: thanos-receiver
          image: thanosio/thanos:v0.25.1
          args:
            - receive
            - --grpc-address=0.0.0.0:10901
            - --http-address=0.0.0.0:10902
            - --remote-write.address=0.0.0.0:19291 # 提供给 prometheus 的 remote_write 端口
            - --receive.replication-factor=3 # 副本数，详细解释参考https://thanos.io/tip/proposals-done/201812-thanos-remote-receive.md/#:~:text=--receive.replication-factor=3
            - --objstore.config-file=/etc/secret/thanos.yaml # 对象存储配置文件
            - --tsdb.path=/var/thanos/receiver # 本地tsdb路径
            - --tsdb.retention=1d # 热数据的保存时间
            - --label=receive_replica="$(NAME)" # 用于过滤重复数据的标签
            - --receive.local-endpoint=$(NAME).thanos-receiver:10901 # 节点endpoint，hashring 中记录的节点host需要与此处保持一致
            - --receive.hashrings-file=/var/lib/thanos-receive/hashring.json # hashring文件，用于记录集群节点
          ports:
            - containerPort: 10901
              name: grpc
            - containerPort: 10902
              name: http
            - containerPort: 19291
              name: remote-write
          env:
            - name: NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
          livenessProbe:
            failureThreshold: 8
            httpGet:
              path: /-/healthy
              port: 10902
              scheme: HTTP
            periodSeconds: 30
          readinessProbe:
            failureThreshold: 20
            httpGet:
              path: /-/ready
              port: 10902
              scheme: HTTP
            periodSeconds: 5
          volumeMounts:
            - mountPath: /var/thanos/receiver
              name: data
              readOnly: false
            - name: hashring-config
              mountPath: /var/lib/thanos-receive
            - name: object-storage-config
              mountPath: /etc/secret
              readOnly: false
      volumes:
        - name: object-storage-config
          secret:
            secretName: thanos-objectstorage
        - name: hashring-config
          configMap:
            name: hashring-config
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes:
          - ReadWriteOnce
        storageClassName: longhorn
        resources:
          requests:
            storage: 2Gi
---
apiVersion: v1
kind: Service
metadata:
  name: thanos-receiver
  namespace: monitoring
spec:
  clusterIP: None
  ports:
    - name: grpc
      port: 10901
      targetPort: 10901
    - name: http
      port: 10902
      targetPort: 10902
    - name: remote-write
      port: 19291
      targetPort: 19291
  selector:
    app: thanos-receiver
root@master:~/prometheus# kubectl apply -f thanos-receiver.yaml

6.4.4 创建 query

root@master:~/prometheus# cat thanos-query.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-querier
  namespace: monitoring
  labels:
    app: thanos-querier
spec:
  selector:
    matchLabels:
      app: thanos-querier
  template:
    metadata:
      labels:
        app: thanos-querier
    spec:
      containers:
        - name: thanos
          image: thanosio/thanos:v0.25.1
          imagePullPolicy: IfNotPresent
          args:
            - query
            - --log.level=debug
            - --query.replica-label=prometheus_replica  # prometheus-operator 里面配置的副本标签为 prometheus_replica
            - --query.replica-label=receive_replica     # receiver 标签
            # Discover local store APIs using DNS SRV.
            - --store=dnssrv+thanos-store-gateway:10901 # store 存储网关
          ports:
            - name: http
              containerPort: 10902
            - name: grpc
              containerPort: 10901

---
apiVersion: v1
kind: Service
metadata:
  name: thanos-querier
  namespace: monitoring
  labels:
    app: thanos-querier
spec:
  ports:
    - port: 9090
      targetPort: http
      name: http
      nodePort: 32700
  selector:
    app: thanos-querier
  type: NodePort
root@master:~/prometheus# kubectl apply thanos-query.yaml

浏览器访问：

6.4.5 配置 Prometheus 对接 receiver

注意：

需要在 Prometheus 的 yaml 中添加 remotewrite 地址

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  labels:
    app.kubernetes.io/component: prometheus
    app.kubernetes.io/instance: k8s
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 2.36.1
  name: k8s
  namespace: monitoring
spec:
  alerting:
    alertmanagers:
    - apiVersion: v2
      name: alertmanager-main
      namespace: monitoring
      port: web
  enableFeatures: []
  externalLabels: {}
  image: quay.io/prometheus/prometheus:v2.36.1
  nodeSelector:
    kubernetes.io/os: linux
  podMetadata:
    labels:
      app.kubernetes.io/component: prometheus
      app.kubernetes.io/instance: k8s
      app.kubernetes.io/name: prometheus
      app.kubernetes.io/part-of: kube-prometheus
      app.kubernetes.io/version: 2.36.1
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  probeNamespaceSelector: {}
  probeSelector: {}
  replicas: 2
  resources:
    requests:
      memory: 400Mi
  ruleSelector: # 用来匹配rule规则的selector
    matchLabels: # 匹配的是具有下面两个标签的PrometheusRule这个资源对象
      prometheus: k8s
      role: alert-rules
  ruleNamespaceSelector: {}
  ruleSelector: {}
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: prometheus-k8s
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  version: 2.36.1
  remoteWrite: #加入remoteWrite地址，注意替换为自己的地址
  - url: http://thanos-receiver:19291/api/v1/receive
root@master:~/prometheus# kubectl apply -f prometheus-prometheus.yaml

query 浏览器数据查询：

后续只需要对接 Grafana 即可视化

暂无评论

发送评论 编辑评论 


				

			

						

				

					

						

							

								
							

							
						

					

				

				

					

						

							

								
							

							
						

					

				

				

					

						

							

								
							

							
							
													

					

				

			

			
			

				

					
				

			

				

											

							
							
						

																				
					
											
						

	

		
|´・ω・)ノ
ヾ(≧∇≦*)ゝ
(☆ω☆)
（╯‵□′）╯︵┴─┴
￣﹃￣
(/ω＼)
∠( ᐛ 」∠)＿
(๑•̀ㅁ•́ฅ)
→_→
୧(๑•̀⌄•́๑)૭
٩(ˊᗜˋ*)و
(ノ°ο°)ノ
(´இ皿இ｀)
⌇●﹏●⌇
(ฅ´ω`ฅ)
(╯°A°)╯︵○○○
φ(￣∇￣o)
ヾ(´･ ･｀｡)ノ"
( ง ᵒ̌皿ᵒ̌)ง⁼³₌₃
(ó﹏ò｡)
Σ(っ °Д °;)っ
( ,,´･ω･)ﾉ"(´っω･｀｡)
╮(╯▽╰)╭
o(*////▽////*)q
＞﹏＜
( ๑´•ω•) "(ㆆᴗㆆ)
😂
😀
😅
😊
🙂
🙃
😌
😍
😘
😜
😝
😏
😒
🙄
😳
😡
😔
😫
😱
😭
💩
👻
🙌
🖕
👍
👫
👬
👭
🌚
🌝
🙈
💊
😶
🙏
🍦
🍉
😣
Source: github.com/k4yt3x/flowerhd
	

	

		
颜文字
Emoji
小恐龙
花!
	


									

			

			
			
		

	






上一篇
下一篇