前言：

在多集群管理的时候可能会出现 Prometheus CRD 版本不知此 header 字段所以无法做到给租户就集群 Prometheus 自定义集群标识，那么就可以通过下面 openresty 来实现通过代理添加标识

整体方案

方案描述：

在服务端部署一个openresty(nginx)，接收 prometheus 的 remote-write 请求，将接收请求路径 ^/THANOS-TENANT/{tenant}/api/v1/receive$ 中的/THANOS-TENANT/{tenant}，写入 header: THANOS-TENANT: {cluster}，然后将请求再转发给 thanos receive，在服务端完成 header 转换流程。而 prometheus 侧 remote-write 配置只需要为集群配置特定的 uri 即可，如

spec:
  remoteWrite:
  - url: http://{{ openresty_endpoint }}/THANOS-TENANT/{tenant}/api/v1/receive

方案优势：

借助 openresty ，部署于 thanos receive 端，充当网关作用，后续服务横向扩展也更方便；
通过 lua 脚本，编程式的配置文件，更加灵活；
不仅能解决 remote-write header 问题，后续如数据准入控制，也可以通过更新配置来控制；
无需对已有环境服务进行变更升级；不受 prometheus-operator 旧版本影响；

关键代码：

    server {
        listen 80;
        resolver coredns.kube-system.svc.cluster.local valid=10s;

        location / {

            # Lua script to set custom header based on the request path
            access_by_lua_block {
                local path = ngx.var.request_uri

            # 如果说在我们的租户集群中Prometheus 是没有在 url 添加标识的话会在 thanos 识别为 unknown
                if string.match(path, "^/api/v1/receive") then
                    ngx.req.set_header("THANOS-TENANT", "unknown")

            # 如果在租户集群中的 Prometheus RW 里面添加了 /THANOS%-TENANT/(%a+)/api/v1/receive 标识，那么就需要将(%a+) 改为集群自定义标识
                elseif string.match(path, "^/THANOS%-TENANT/(%a+)/api/v1/receive$") then
                    ngx.req.set_header("THANOS-TENANT", string.match(path, "^/THANOS%-TENANT/(%a+)/api/v1/receive$"))
                    ngx.req.set_uri("/api/v1/receive")
                end
            }

            # Service FQDN thanos配置部分
            proxy_pass http://thanos-receive-nodeport.openresty.svc.cluster.local:19291;    
            proxy_ignore_client_abort on;
            proxy_send_timeout 30s;
            proxy_read_timeout 30s;

        }
    }

配置流程

使用如下配置文件部署 openresty 服务：

kind: ConfigMap
apiVersion: v1
metadata:
  name: openresty-config
  namespace: openresty
data:
  nginx.conf: >-
    # nginx.conf  --  docker-openresty

    #

    # This file is installed to:

    #   `/usr/local/openresty/nginx/conf/nginx.conf`

    # and is the file loaded by nginx at startup,

    # unless the user specifies otherwise.

    #

    # It tracks the upstream OpenResty's `nginx.conf`, but removes the `server`

    # section and adds this directive:

    #     `include /etc/nginx/conf.d/*.conf;`

    #

    # The `docker-openresty` file `nginx.vh.default.conf` is copied to

    # `/etc/nginx/conf.d/default.conf`.  It contains the `server section

    # of the upstream `nginx.conf`.

    #

    # See
    https://github.com/openresty/docker-openresty/blob/master/README.md#nginx-config-files

    #

    #user  nobody;

    #worker_processes 1;

    # Enables the use of JIT for regular expressions to speed-up their
    processing.

    pcre_jit on;

    #error_log  logs/error.log;

    #error_log  logs/error.log  notice;

    #error_log  logs/error.log  info;

    #pid        logs/nginx.pid;

    events {
        worker_connections  1024;
    }

    http {
        include       mime.types;
        default_type  application/octet-stream;

        server {
            listen 80;
            resolver coredns.kube-system.svc.cluster.local valid=10s;

            location / {

                # Lua script to set custom header based on the request path
                access_by_lua_block {
                    local path = ngx.var.request_uri

                    if string.match(path, "^/api/v1/receive") then
                        ngx.req.set_header("THANOS-TENANT", "unknown")

                    elseif string.match(path, "^/THANOS%-TENANT/(%a+)/api/v1/receive$") then
                        ngx.req.set_header("THANOS-TENANT", string.match(path, "^/THANOS%-TENANT/(%a+)/api/v1/receive$"))
                        ngx.req.set_uri("/api/v1/receive")
                    end
                }

                proxy_pass http://thanos-receive-nodeport.openresty.svc.cluster.local:19291;
                proxy_ignore_client_abort on;
                proxy_send_timeout 30s;
                proxy_read_timeout 30s;

            }
        }

        # Enables or disables the use of underscores in client request header fields.
        # When the use of underscores is disabled, request header fields whose names contain underscores are marked as invalid and become subject to the ignore_invalid_headers directive.
        # underscores_in_headers off;

        #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
        #                  '$status $body_bytes_sent "$http_referer" '
        #                  '"$http_user_agent" "$http_x_forwarded_for"';

        #access_log  logs/access.log  main;

            # Log in JSON Format
            # log_format nginxlog_json escape=json '{ "timestamp": "$time_iso8601", '
            # '"remote_addr": "$remote_addr", '
            #  '"body_bytes_sent": $body_bytes_sent, '
            #  '"request_time": $request_time, '
            #  '"response_status": $status, '
            #  '"request": "$request", '
            #  '"request_method": "$request_method", '
            #  '"host": "$host",'
            #  '"upstream_addr": "$upstream_addr",'
            #  '"http_x_forwarded_for": "$http_x_forwarded_for",'
            #  '"http_referrer": "$http_referer", '
            #  '"http_user_agent": "$http_user_agent", '
            #  '"http_version": "$server_protocol", '
            #  '"nginx_access": true }';
            # access_log /dev/stdout nginxlog_json;

        # See Move default writable paths to a dedicated directory (#119)
        # https://github.com/openresty/docker-openresty/issues/119
        client_body_temp_path /var/run/openresty/nginx-client-body;
        proxy_temp_path       /var/run/openresty/nginx-proxy;
        fastcgi_temp_path     /var/run/openresty/nginx-fastcgi;
        uwsgi_temp_path       /var/run/openresty/nginx-uwsgi;
        scgi_temp_path        /var/run/openresty/nginx-scgi;

        sendfile        on;
        #tcp_nopush     on;

        keepalive_timeout  30;

        #gzip  on;

        include /etc/nginx/conf.d/*.conf;

        # Don't reveal OpenResty version to clients.
        # server_tokens off;
    }

kind: Deployment
apiVersion: apps/v1
metadata:
  name: openresty
  namespace: openresty
  labels:
    app: openresty
spec:
  replicas: 1
  selector:
    matchLabels:
      app: openresty
  template:
    metadata:
      labels:
        app: openresty
    spec:
      volumes:
        - name: openresty-config
          configMap:
            name: openresty-config
            defaultMode: 420
      containers:
        - name: openresty
          image: 'openresty/openresty:1.21.4.1-0-alpine'
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
          resources: {}
          volumeMounts:
            - name: openresty-config
              mountPath: /usr/local/openresty/nginx/conf/nginx.conf
              subPath: nginx.conf
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: IfNotPresent
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      dnsPolicy: ClusterFirst
      serviceAccountName: default
      serviceAccount: default
      securityContext: {}
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 25%
      maxSurge: 25%
  revisionHistoryLimit: 10
  progressDeadlineSeconds: 600

kind: Service
apiVersion: v1
metadata:
  name: openresty
  namespace: openresty
  labels:
    app: openresty
spec:
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 80
  selector:
    app: openresty
  type: ClusterIP
  sessionAffinity: None
  ipFamilies:
    - IPv4
  ipFamilyPolicy: SingleStack
  internalTrafficPolicy: Cluster

方案验证

通过如下命令在 Prometheus 的CR 中修改 remote-write 的配置

kubectl edit prometheus -n kubesphere-monitoring-system

spec:
  remoteWrite:
  # 下面两个 rul 二选一，第一个 rul 显示为 unknown，第二个 url 自定义为集群标识如 dev
  - url: http://openresty.openresty.svc:80/api/v1/receive
  - url: http://openresty.openresty.svc:80/THANOS-TENANT/dev/api/v1/receive

  ... ...

查看 Prometheus/Thanos 日志，数据分别写入对用 Thanos 租户中

prometheus 日志

ts=2023-07-26T07:25:37.295Z caller=dedupe.go:112 component=remote level=info remote_name=01b2c3 url=http://openresty.openresty.svc:80/api/v1/receive msg="Done replaying WAL" duration=1.024388484s
ts=2023-07-26T07:25:37.300Z caller=dedupe.go:112 component=remote level=info remote_name=f1cb32 url=http://openresty.openresty.svc:80/THANOS-TENANT/dev/api/v1/receive msg="Done replaying WAL" duration=1.028902905s

thanos receive 日志

level=info ts=2023-07-26T07:49:58.844162217Z caller=head.go:685 component=receive component=multi-tsdb tenant=unknown msg="WAL segment loaded" segment=0 maxSegment=0
level=info ts=2023-07-26T07:49:58.844236187Z caller=head.go:722 component=receive component=multi-tsdb tenant=unknown msg="WAL replay completed" checkpoint_replay_duration=224.829µs wal_replay_duration=2.700177ms wbl_replay_duration=132ns total_replay_duration=3.141629ms
level=info ts=2023-07-26T07:49:58.860589307Z caller=head.go:685 component=receive component=multi-tsdb tenant=dev msg="WAL segment loaded" segment=0 maxSegment=0
level=info ts=2023-07-26T07:49:58.860648193Z caller=head.go:722 component=receive component=multi-tsdb tenant=dev msg="WAL replay completed" checkpoint_replay_duration=70.569µs wal_replay_duration=3.725675ms wbl_replay_duration=315ns total_replay_duration=3.834049ms

2025年 4月
一	二	三	四	五	六	日
	1	2	3	4	5	6
7	8	9	10	11	12	13
14	15	16	17	18	19	20
21	22	23	24	25	26	27
28	29	30

前言：

整体方案

配置流程

方案验证

发送评论 编辑评论

发送评论编辑评论