update helm charts w. irate + drop vector(0) on CPU + Memory HPA

Author: dmwilson-grafana

operations/helm/charts/mimir-distributed/templates/distributor/distributor-so.yaml

@@ -27,7 +27,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
+      query: max_over_time(sum(sum by (pod) (irate(container_cpu_usage_seconds_total{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $cpu_request := dig "requests" "cpu" nil .Values.distributor.resources }}
       threshold: {{ mulf (include "mimir.cpuToMilliCPU" (dict "value" $cpu_request)) (divf .Values.distributor.kedaAutoscaling.targetCPUUtilizationPercentage 100) | floor | int64 | quote }}
@@ -45,7 +45,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="distributor",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0))
+      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="distributor",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="distributor",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0)))
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $mem_request := dig "requests" "memory" nil .Values.distributor.resources }}
       threshold: {{ mulf (include "mimir.siToBytes" (dict "value" $mem_request)) (divf .Values.distributor.kedaAutoscaling.targetMemoryUtilizationPercentage 100) | floor | int64 | quote }}

operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-so.yaml

@@ -27,7 +27,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
+      query: max_over_time(sum(sum by (pod) (irate(container_cpu_usage_seconds_total{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $cpu_request := dig "requests" "cpu" nil .Values.query_frontend.resources }}
       threshold: {{ mulf (include "mimir.cpuToMilliCPU" (dict "value" $cpu_request)) (divf .Values.query_frontend.kedaAutoscaling.targetCPUUtilizationPercentage 100) | floor | int64 | quote }}
@@ -45,7 +45,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="query-frontend",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="query-frontend",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0))
+      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="query-frontend",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="query-frontend",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0)))
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $mem_request := dig "requests" "memory" nil .Values.query_frontend.resources }}
       threshold: {{ mulf (include "mimir.siToBytes" (dict "value" $mem_request)) (divf .Values.query_frontend.kedaAutoscaling.targetMemoryUtilizationPercentage 100) | floor | int64 | quote }}

operations/helm/charts/mimir-distributed/templates/ruler-query-frontend/ruler-query-frontend-so.yaml

@@ -28,7 +28,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
+      query: max_over_time(sum(sum by (pod) (irate(container_cpu_usage_seconds_total{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $cpu_request := dig "requests" "cpu" nil .Values.ruler_query_frontend.resources }}
       threshold: {{ mulf (include "mimir.cpuToMilliCPU" (dict "value" $cpu_request)) (divf .Values.ruler_query_frontend.kedaAutoscaling.targetCPUUtilizationPercentage 100) | floor | int64 | quote }}
@@ -46,7 +46,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0))
+      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler-query-frontend",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0)))
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $mem_request := dig "requests" "memory" nil .Values.ruler_query_frontend.resources }}
       threshold: {{ mulf (include "mimir.siToBytes" (dict "value" $mem_request)) (divf .Values.ruler_query_frontend.kedaAutoscaling.targetMemoryUtilizationPercentage 100) | floor | int64 | quote }}

operations/helm/charts/mimir-distributed/templates/ruler/ruler-so.yaml

@@ -27,7 +27,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
+      query: max_over_time(sum(sum by (pod) (irate(container_cpu_usage_seconds_total{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[5m])) and max by (pod) (up{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0)[15m:]) * 1000
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $cpu_request := dig "requests" "cpu" nil .Values.ruler.resources }}
       threshold: {{ mulf (include "mimir.cpuToMilliCPU" (dict "value" $cpu_request)) (divf .Values.ruler.kedaAutoscaling.targetCPUUtilizationPercentage 100) | floor | int64 | quote }}
@@ -45,7 +45,7 @@ spec:
       {{- if .Values.kedaAutoscaling.authentication.enabled }}
       authModes: "{{ .Values.kedaAutoscaling.authentication.authModes }}"
       {{- end }}
-      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="ruler",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0))
+      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) and max by (pod) (up{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) > 0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="ruler",namespace="{{ .Release.Namespace }}", resource="memory",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="ruler",namespace="{{ .Release.Namespace }}",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler",namespace="{{ .Release.Namespace }}", reason="OOMKilled",{{ include "toPromQLLabelSelector" .Values.kedaAutoscaling.toPromQLLabelSelector }}}) or vector(0)))
       serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
       {{- $mem_request := dig "requests" "memory" nil .Values.ruler.resources }}
       threshold: {{ mulf (include "mimir.siToBytes" (dict "value" $mem_request)) (divf .Values.ruler.kedaAutoscaling.targetMemoryUtilizationPercentage 100) | floor | int64 | quote }}

operations/helm/tests/keda-autoscaling-global-values-generated/mimir-distributed/templates/distributor/distributor-so.yaml

@@ -31,7 +31,7 @@ spec:
   triggers:
   - metadata:
       authModes: "basic"
-      query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="distributor",namespace="citestns",cluster="my-cluster-name"}[5m])) and max by (pod) (up{container="distributor",namespace="citestns",cluster="my-cluster-name"}) > 0)[15m:]) * 1000
+      query: max_over_time(sum(sum by (pod) (irate(container_cpu_usage_seconds_total{container="distributor",namespace="citestns",cluster="my-cluster-name"}[5m])) and max by (pod) (up{container="distributor",namespace="citestns",cluster="my-cluster-name"}) > 0)[15m:]) * 1000
       serverAddress: https://mimir.example.com/prometheus
       threshold: "80"
       customHeaders: "X-Scope-OrgID=tenant"
@@ -42,7 +42,7 @@ spec:
         name: "keda-triggger-auth"
   - metadata:
       authModes: "basic"
-      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="distributor",namespace="citestns",cluster="my-cluster-name"}) and max by (pod) (up{container="distributor",namespace="citestns",cluster="my-cluster-name"}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="distributor",namespace="citestns", resource="memory",cluster="my-cluster-name"}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="distributor",namespace="citestns",cluster="my-cluster-name"}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor",namespace="citestns", reason="OOMKilled",cluster="my-cluster-name"}) or vector(0))
+      query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="distributor",namespace="citestns",cluster="my-cluster-name"}) and max by (pod) (up{container="distributor",namespace="citestns",cluster="my-cluster-name"}) > 0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="distributor",namespace="citestns", resource="memory",cluster="my-cluster-name"}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="distributor",namespace="citestns",cluster="my-cluster-name"}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor",namespace="citestns", reason="OOMKilled",cluster="my-cluster-name"}) or vector(0)))
       serverAddress: https://mimir.example.com/prometheus
       threshold: "429496729"
       customHeaders: "X-Scope-OrgID=tenant"