Merge pull request #783 from Adirael/feature/support-json-eval

Add support for ajson scripting engine

Author: demonCoder95

README.md

@@ -116,6 +116,7 @@ metadata:
   annotations:
     # metric-config.<metricType>.<metricName>.<collectorType>/<configKey>
     metric-config.pods.requests-per-second.json-path/json-key: "$.http_server.rps"
+    metric-config.pods.requests-per-second.json-path/json-eval: "ceil($['active processes'] / $['total processes'] * 100)" # cannot use both json-eval and json-key
     metric-config.pods.requests-per-second.json-path/path: /metrics
     metric-config.pods.requests-per-second.json-path/port: "9090"
     metric-config.pods.requests-per-second.json-path/scheme: "https"
@@ -158,6 +159,10 @@ The json-path query support depends on the
 See the README for possible queries. It's expected that the metric you query
 returns something that can be turned into a `float64`.
 
+The `json-eval` configuration option allows for more complex calculations to be
+performed on the extracted metric. The `json-eval` expression is evaluated using
+[ajson's script engine](https://github.com/spyzhov/ajson?tab=readme-ov-file#script-engine).
+
 The other configuration options `path`, `port` and `scheme` specify where the metrics
 endpoint is exposed on the pod. The `path` and `port` options do not have default values
 so they must be defined. The `scheme` is optional and defaults to `http`.
@@ -825,6 +830,7 @@ metadata:
   annotations:
     # metric-config.<metricType>.<metricName>.<collectorType>/<configKey>
     metric-config.external.unique-metric-name.json-path/json-key: "$.some-metric.value"
+    metric-config.external.unique-metric-name.json-path/json-eval: ceil($['active processes'] / $['total processes'] * 100) # cannot use both json-eval and json-key
     metric-config.external.unique-metric-name.json-path/endpoint: "http://metric-source.app-namespace:8080/metrics"
     metric-config.external.unique-metric-name.json-path/aggregator: "max"
     metric-config.external.unique-metric-name.json-path/interval: "60s" # optional
@@ -852,6 +858,8 @@ The HTTP collector similar to the Pod Metrics collector. The following
 configuration values are supported:
 
 - `json-key` to specify the JSON path of the metric to be queried
+- `json-eval` to specify an evaluate string to [evaluate on the script engine](https://github.com/spyzhov/ajson?tab=readme-ov-file#script-engine), 
+    cannot be used in conjunction with `json-key`
 - `endpoint` the fully formed path to query for the metric. In the above example a Kubernetes _Service_
     in the namespace `app-namespace` is called.
 - `aggregator` is only required if the metric is an array of values and specifies how the values

pkg/collector/http_collector.go

@@ -19,6 +19,7 @@ const (
 	HTTPMetricNameLegacy      = "http"
 	HTTPEndpointAnnotationKey = "endpoint"
 	HTTPJsonPathAnnotationKey = "json-key"
+	HTTPJsonEvalAnnotationKey = "json-eval"
 )
 
 type HTTPCollectorPlugin struct{}
@@ -31,14 +32,27 @@ func (p *HTTPCollectorPlugin) NewCollector(_ context.Context, hpa *autoscalingv2
 	collector := &HTTPCollector{
 		namespace: hpa.Namespace,
 	}
+
 	var (
 		value string
 		ok    bool
+
+		jsonPath string
+		jsonEval string
 	)
-	if value, ok = config.Config[HTTPJsonPathAnnotationKey]; !ok {
-		return nil, fmt.Errorf("config value %s not found", HTTPJsonPathAnnotationKey)
+
+	if value, ok = config.Config[HTTPJsonPathAnnotationKey]; ok {
+		jsonPath = value
+	}
+	if value, ok = config.Config[HTTPJsonEvalAnnotationKey]; ok {
+		jsonEval = value
+	}
+	if jsonPath == "" && jsonEval == "" {
+		return nil, fmt.Errorf("config value %s or %s not found", HTTPJsonPathAnnotationKey, HTTPJsonEvalAnnotationKey)
+	}
+	if jsonPath != "" && jsonEval != "" {
+		return nil, fmt.Errorf("config value %s and %s cannot be used together", HTTPJsonPathAnnotationKey, HTTPJsonEvalAnnotationKey)
 	}
-	jsonPath := value
 
 	if value, ok = config.Config[HTTPEndpointAnnotationKey]; !ok {
 		return nil, fmt.Errorf("config value %s not found", HTTPEndpointAnnotationKey)
@@ -62,7 +76,7 @@ func (p *HTTPCollectorPlugin) NewCollector(_ context.Context, hpa *autoscalingv2
 			return nil, err
 		}
 	}
-	jsonPathGetter, err := httpmetrics.NewJSONPathMetricsGetter(httpmetrics.DefaultMetricsHTTPClient(), aggFunc, jsonPath)
+	jsonPathGetter, err := httpmetrics.NewJSONPathMetricsGetter(httpmetrics.DefaultMetricsHTTPClient(), aggFunc, jsonPath, jsonEval)
 	if err != nil {
 		return nil, err
 	}

pkg/collector/httpmetrics/json_path.go

@@ -16,18 +16,22 @@ import (
 // the json path query.
 type JSONPathMetricsGetter struct {
 	jsonPath   string
+	jsonEval   string
 	aggregator AggregatorFunc
 	client     *http.Client
 }
 
 // NewJSONPathMetricsGetter initializes a new JSONPathMetricsGetter.
-func NewJSONPathMetricsGetter(httpClient *http.Client, aggregatorFunc AggregatorFunc, jsonPath string) (*JSONPathMetricsGetter, error) {
+func NewJSONPathMetricsGetter(httpClient *http.Client, aggregatorFunc AggregatorFunc, jsonPath string, jsonEval string) (*JSONPathMetricsGetter, error) {
 	// check that jsonPath parses
-	_, err := ajson.ParseJSONPath(jsonPath)
-	if err != nil {
-		return nil, err
+	if jsonPath != "" {
+		_, err := ajson.ParseJSONPath(jsonPath)
+		if err != nil {
+			return nil, err
+		}
 	}
-	return &JSONPathMetricsGetter{client: httpClient, aggregator: aggregatorFunc, jsonPath: jsonPath}, nil
+
+	return &JSONPathMetricsGetter{client: httpClient, aggregator: aggregatorFunc, jsonPath: jsonPath, jsonEval: jsonEval}, nil
 }
 
 var DefaultRequestTimeout = 15 * time.Second
@@ -67,9 +71,19 @@ func (g *JSONPathMetricsGetter) GetMetric(metricsURL url.URL) (float64, error) {
 		return 0, err
 	}
 
-	nodes, err := root.JSONPath(g.jsonPath)
-	if err != nil {
-		return 0, err
+	var nodes []*ajson.Node
+	if g.jsonPath != "" {
+		nodes, err = root.JSONPath(g.jsonPath)
+		if err != nil {
+			return 0, err
+		}
+	} else {
+		result, err := ajson.Eval(root, g.jsonEval)
+		nodes = append(nodes, result)
+
+		if err != nil {
+			return 0, err
+		}
 	}
 
 	if len(nodes) == 0 {

pkg/collector/httpmetrics/json_path_test.go

@@ -26,6 +26,7 @@ func TestJSONPathMetricsGetter(t *testing.T) {
 		name         string
 		jsonResponse []byte
 		jsonPath     string
+		jsonEval     string
 		result       float64
 		aggregator   AggregatorFunc
 		err          error
@@ -58,6 +59,19 @@ func TestJSONPathMetricsGetter(t *testing.T) {
 			result:       5,
 			aggregator:   Average,
 		},
+		{
+			name:         "evaluated script",
+			jsonResponse: []byte(`{"active processes":1,"total processes":10}`),
+			jsonEval:     "ceil($['active processes'] / $['total processes'] * 100)",
+			result:       10,
+			aggregator:   Average,
+		},
+		{
+			name:         "invalid script should error",
+			jsonResponse: []byte(`{"active processes":1,"total processes":10}`),
+			jsonEval:     "ceil($['active processes'] ) $['total processes'] * 100)",
+			err:          errors.New("wrong request: formula has no left parentheses"),
+		},
 		{
 			name:         "json path not resulting in array or number should lead to error",
 			jsonResponse: []byte(`{"metric.value":5}`),
@@ -74,7 +88,7 @@ func TestJSONPathMetricsGetter(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			server := makeTestHTTPServer(t, tc.jsonResponse)
 			defer server.Close()
-			getter, err := NewJSONPathMetricsGetter(DefaultMetricsHTTPClient(), tc.aggregator, tc.jsonPath)
+			getter, err := NewJSONPathMetricsGetter(DefaultMetricsHTTPClient(), tc.aggregator, tc.jsonPath, tc.jsonEval)
 			require.NoError(t, err)
 			url, err := url.Parse(fmt.Sprintf("%s/metrics", server.URL))
 			require.NoError(t, err)

pkg/collector/httpmetrics/pod_metrics.go

@@ -33,6 +33,7 @@ func NewPodMetricsJSONPathGetter(config map[string]string) (*PodMetricsJSONPathG
 	getter := PodMetricsJSONPathGetter{}
 	var (
 		jsonPath   string
+		jsonEval   string
 		aggregator AggregatorFunc
 		err        error
 	)
@@ -41,6 +42,16 @@ func NewPodMetricsJSONPathGetter(config map[string]string) (*PodMetricsJSONPathG
 		jsonPath = v
 	}
 
+	if v, ok := config["json-eval"]; ok {
+		jsonEval = v
+	}
+
+	if jsonPath == "" && jsonEval == "" {
+		return nil, fmt.Errorf("config value json-key or json-eval must be set")
+	} else if jsonPath != "" && jsonEval != "" {
+		return nil, fmt.Errorf("config value json-key and json-eval are mutually exclusive")
+	}
+
 	if v, ok := config["scheme"]; ok {
 		getter.scheme = v
 	}
@@ -93,7 +104,7 @@ func NewPodMetricsJSONPathGetter(config map[string]string) (*PodMetricsJSONPathG
 		connectTimeout = d
 	}
 
-	jsonPathGetter, err := NewJSONPathMetricsGetter(CustomMetricsHTTPClient(requestTimeout, connectTimeout), aggregator, jsonPath)
+	jsonPathGetter, err := NewJSONPathMetricsGetter(CustomMetricsHTTPClient(requestTimeout, connectTimeout), aggregator, jsonPath, jsonEval)
 	if err != nil {
 		return nil, err
 	}

pkg/collector/httpmetrics/pod_metrics_test.go

@@ -86,6 +86,17 @@ func TestNewPodJSONPathMetricsGetter(t *testing.T) {
 		port:         9090,
 		rawQuery:     "foo=bar&baz=bop",
 	}, getterWithRawQuery)
+
+	configErrorMixedPathEval := map[string]string{
+		"json-key": "{}",
+		"json-eval": "avg($.values)",
+		"scheme":   "http",
+		"path":     "/metrics",
+		"port":     "9090",
+	}
+
+	_, err6 := NewPodMetricsJSONPathGetter(configErrorMixedPathEval)
+	require.Error(t, err6)
 }
 
 func TestBuildMetricsURL(t *testing.T) {