[Celerdata] Add a grok expression to parse the new log format of BE (ECOINT-101) (DataDog#2603)

yandongxiao · dd-dominic · web-flow · commit 9907ec90f780 · 2025-04-08T14:59:14.000Z
* [Celerdata] Add a grok expression to parse the new log format of BE

Signed-off-by: yandongxiao &lt;dxyan06@gmail.com&gt;

* Update celerdata/CHANGELOG.md

Co-authored-by: Dominic Medina &lt;115744456+dd-dominic@users.noreply.github.com&gt;

* Update celerdata/README.md

Co-authored-by: Dominic Medina &lt;115744456+dd-dominic@users.noreply.github.com&gt;

* Update celerdata/datadog_checks/celerdata/__about__.py

Co-authored-by: Dominic Medina &lt;115744456+dd-dominic@users.noreply.github.com&gt;

* run command: ddev validate models celerdata -s

Signed-off-by: yandongxiao &lt;dxyan06@gmail.com&gt;

* update datadog_checks/celerdata/config_models/instance.py

Signed-off-by: yandongxiao &lt;dxyan06@gmail.com&gt;

* update datadog_checks/celerdata/data/conf.yaml.example

Signed-off-by: yandongxiao &lt;dxyan06@gmail.com&gt;

* Update license information

Signed-off-by: yandongxiao &lt;dxyan06@gmail.com&gt;

* add test for new rules

Signed-off-by: yandongxiao &lt;dxyan06@gmail.com&gt;

---------

Signed-off-by: yandongxiao &lt;dxyan06@gmail.com&gt;
Co-authored-by: Dominic Medina &lt;115744456+dd-dominic@users.noreply.github.com&gt;
diff --git a/celerdata/CHANGELOG.md b/celerdata/CHANGELOG.md
@@ -1,5 +1,11 @@
 # CHANGELOG - celerdata
 
+## 1.1.0 / 2025-02-21
+
+***Added***:
+
+* Add a grok expression to parse the new log format of BE
+
 ## 1.0.0 / 2024-03-11
 
 ***Added***:
diff --git a/celerdata/README.md b/celerdata/README.md
@@ -18,7 +18,7 @@ To collect StarRocks [metrics][9] and logs:
 2. Install the CelerData check on your host with the following command:
 
    ```shell
-   datadog-agent integration install -t datadog-celerdata==1.0.0
+   datadog-agent integration install -t datadog-celerdata==1.1.0
    ```
 
 ### Configuration
diff --git a/celerdata/assets/logs/celerdata.yaml b/celerdata/assets/logs/celerdata.yaml
@@ -18,6 +18,7 @@ pipeline:
         - 2024-03-28 18:58:43,380 INFO (nioEventLoopGroup-4-3|154) [RestBaseAction.handleRequest():73] receive http request. url=/api/health
         - 2024-04-10 10:32:23.937+08:00 INFO (stateChangeExecutor|95) [StateChangeExecutor.runOneCycle():179] finished to transfer FE type from INIT to LEADER
         - I0328 10:22:09.303526 1066 olap_server.cpp:844] try to perform path gc by tablet!
+        - I20250211 13:17:55.023299 127358726047296 query_context.cpp:60] finished query_id:837ca8a5-e8a4-11ef-89f3-0e809d90bae3 context life time:1476324 cpu costs:428108 peak memusage:298504 scan_bytes:5139 spilled bytes:0
       grok:
         supportRules: fe_other_log %{word:loglevel} \(%{notSpace}\|%{integer}\) \[%{word:class_name}.%{word:method_name}\(\):%{integer:file_line}\] %{data:message}
         matchRules: >-
@@ -26,6 +27,8 @@ pipeline:
           fe.log2 %{date("yyyy-MM-dd' 'HH:mm:ss.SSSZZ"):timestamp} %{fe_other_log}
 
           be.info %{regex("\\w"):loglevel}%{date("MMdd HH:mm:ss.SSSSSS"):timestamp}\s+%{integer:thread_id} %{notSpace:file_name}\:%{integer:file_line}\] %{data:message}
+
+          be.info2 %{regex("\\w"):loglevel}%{date("yyyyMMdd HH:mm:ss.SSSSSS"):timestamp}\s+%{integer:thread_id} %{notSpace:file_name}\:%{integer:file_line}\] %{data:message}
     - type: status-remapper
       name: Define `loglevel` as the official status of the log
       enabled: true
diff --git a/celerdata/assets/logs/celerdata_tests.yaml b/celerdata/assets/logs/celerdata_tests.yaml
@@ -42,3 +42,31 @@ tests:
      tags:
        - "source:LOGS_SOURCE"
      timestamp: 1712716343937
+ -
+   sample: "I0211 13:17:55.023299 127358726047296 query_context.cpp:60] finished query_id:837ca8a5-e8a4-11ef-89f3-0e809d90bae3 context life time:1476324 cpu costs:428108 peak memusage:298504 scan_bytes:5139 spilled bytes:0"
+   result:
+     custom:
+       file_line: 60
+       file_name: "query_context.cpp"
+       loglevel: "I"
+       thread_id: 127358726047296
+       timestamp: 35126275023
+     message: "finished query_id:837ca8a5-e8a4-11ef-89f3-0e809d90bae3 context life time:1476324 cpu costs:428108 peak memusage:298504 scan_bytes:5139 spilled bytes:0"
+     status: "info"
+     tags:
+      - "source:LOGS_SOURCE"
+     timestamp: 35126275023
+ -
+   sample: "I20250211 13:17:55.023299 127358726047296 query_context.cpp:60] finished query_id:837ca8a5-e8a4-11ef-89f3-0e809d90bae3 context life time:1476324 cpu costs:428108 peak memusage:298504 scan_bytes:5139 spilled bytes:0"
+   result:
+     custom:
+       file_line: 60
+       file_name: "query_context.cpp"
+       loglevel: "I"
+       thread_id: 127358726047296
+       timestamp: 1739279875023
+     message: "finished query_id:837ca8a5-e8a4-11ef-89f3-0e809d90bae3 context life time:1476324 cpu costs:428108 peak memusage:298504 scan_bytes:5139 spilled bytes:0"
+     status: "info"
+     tags:
+      - "source:LOGS_SOURCE"
+     timestamp: 1739279875023
diff --git a/celerdata/datadog_checks/celerdata/__about__.py b/celerdata/datadog_checks/celerdata/__about__.py
@@ -1 +1 @@
-__version__ = '1.0.0'
+__version__ = '1.1.0'
diff --git a/celerdata/datadog_checks/celerdata/config_models/instance.py b/celerdata/datadog_checks/celerdata/config_models/instance.py
@@ -25,7 +25,7 @@ class AuthToken(BaseModel):
     writer: Optional[MappingProxyType[str, Any]] = None
 
 
-class ExtraMetric(BaseModel):
+class ExtraMetrics(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
         extra='allow',
@@ -44,7 +44,7 @@ class MetricPatterns(BaseModel):
     include: Optional[tuple[str, ...]] = None
 
 
-class Metric(BaseModel):
+class Metrics(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
         extra='allow',
@@ -64,7 +64,7 @@ class Proxy(BaseModel):
     no_proxy: Optional[tuple[str, ...]] = None
 
 
-class ShareLabel(BaseModel):
+class ShareLabels(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
         frozen=True,
@@ -97,7 +97,7 @@ class InstanceConfig(BaseModel):
     exclude_metrics: Optional[tuple[str, ...]] = None
     exclude_metrics_by_labels: Optional[MappingProxyType[str, Union[bool, tuple[str, ...]]]] = None
     extra_headers: Optional[MappingProxyType[str, Any]] = None
-    extra_metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, ExtraMetric]]], ...]] = None
+    extra_metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, ExtraMetrics]]], ...]] = None
     headers: Optional[MappingProxyType[str, Any]] = None
     histogram_buckets_as_distributions: Optional[bool] = None
     hostname_format: Optional[str] = None
@@ -114,7 +114,7 @@ class InstanceConfig(BaseModel):
     kerberos_principal: Optional[str] = None
     log_requests: Optional[bool] = None
     metric_patterns: Optional[MetricPatterns] = None
-    metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, Metric]]], ...]] = None
+    metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, Metrics]]], ...]] = None
     min_collection_interval: Optional[float] = None
     namespace: Optional[str] = Field(None, pattern='\\w*')
     non_cumulative_histogram_buckets: Optional[bool] = None
@@ -129,14 +129,15 @@ class InstanceConfig(BaseModel):
     rename_labels: Optional[MappingProxyType[str, Any]] = None
     request_size: Optional[float] = None
     service: Optional[str] = None
-    share_labels: Optional[MappingProxyType[str, Union[bool, ShareLabel]]] = None
+    share_labels: Optional[MappingProxyType[str, Union[bool, ShareLabels]]] = None
     skip_proxy: Optional[bool] = None
     tag_by_endpoint: Optional[bool] = None
     tags: Optional[tuple[str, ...]] = None
     telemetry: Optional[bool] = None
     timeout: Optional[float] = None
     tls_ca_cert: Optional[str] = None
     tls_cert: Optional[str] = None
+    tls_ciphers: Optional[tuple[str, ...]] = None
     tls_ignore_warning: Optional[bool] = None
     tls_private_key: Optional[str] = None
     tls_protocols_allowed: Optional[tuple[str, ...]] = None
diff --git a/celerdata/datadog_checks/celerdata/data/conf.yaml.example b/celerdata/datadog_checks/celerdata/data/conf.yaml.example
@@ -500,6 +500,16 @@ instances:
     #   - TLSv1.2
     #   - TLSv1.3
 
+    ## @param tls_ciphers - list of strings - optional
+    ## The list of ciphers suites to use when connecting to an endpoint. If not specified, 
+    ## `ALL` ciphers are used. For list of ciphers see: 
+    ## https://www.openssl.org/docs/man1.0.2/man1/ciphers.html
+    #
+    # tls_ciphers:
+    #   - TLS_AES_256_GCM_SHA384
+    #   - TLS_CHACHA20_POLY1305_SHA256
+    #   - TLS_AES_128_GCM_SHA256
+
     ## @param headers - mapping - optional
     ## The headers parameter allows you to send specific headers with every request.
     ## You can use it for explicitly specifying the host header or adding headers for

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = '1.0.0'`
	`1`	`+__version__ = '1.1.0'`