grafana
diff --git a/‎cmd/mimir/config-descriptor.json
Lines changed: 556 additions & 0 deletions b/‎cmd/mimir/config-descriptor.json
Lines changed: 556 additions & 0 deletions
diff --git a/‎cmd/mimir/help-all.txt.tmpl
Lines changed: 92 additions & 0 deletions b/‎cmd/mimir/help-all.txt.tmpl
Lines changed: 92 additions & 0 deletions
diff --git a/‎cmd/mimir/help.txt.tmpl
Lines changed: 20 additions & 0 deletions b/‎cmd/mimir/help.txt.tmpl
Lines changed: 20 additions & 0 deletions
diff --git a/‎development/mimir-microservices-mode/docker-compose.jsonnet
Lines changed: 14 additions & 0 deletions b/‎development/mimir-microservices-mode/docker-compose.jsonnet
Lines changed: 14 additions & 0 deletions
diff --git a/‎docs/sources/mimir/configure/configuration-parameters/index.md
Lines changed: 141 additions & 0 deletions b/‎docs/sources/mimir/configure/configuration-parameters/index.md
Lines changed: 141 additions & 0 deletions
@@ -609,6 +609,12 @@ Usage of ./cmd/mimir/mimir:
     	How long to cache list of blocks for each tenant. (default 5m0s)
   -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl duration
     	How long to cache list of tenants in the bucket. (default 15m0s)
+  -blocks-storage.bucket-store.parquet-max-chunk-size-bytes uint
+    	Maximum size in bytes that can be fetched from the parquet chunks file. If the size exceeds this value the query will stop with limit error.
+  -blocks-storage.bucket-store.parquet-max-data-size-bytes uint
+    	Maximum size in bytes that can be fetched from the parquet labels and chunks data files combined in a single query. If the size exceeds this value the query will stop with limit error.
+  -blocks-storage.bucket-store.parquet-max-row-count uint
+    	Maximum number of rows in a parquet file. If the number of rows exceeds this value the query will stop with limit error.
   -blocks-storage.bucket-store.partitioner-max-gap-bytes uint
     	Max size - in bytes - of a gap for which the partitioner aggregates together two bucket GET object requests. (default 524288)
   -blocks-storage.bucket-store.posting-offsets-in-mem-sampling int
@@ -1969,6 +1975,90 @@ Usage of ./cmd/mimir/mimir:
     	Maximum time to wait for ring stability at startup. If the overrides-exporter ring keeps changing after this period of time, it will start anyway. (default 5m0s)
   -overrides-exporter.ring.wait-stability-min-duration duration
     	Minimum time to wait for ring stability at startup, if set to positive value. Set to 0 to disable.
+  -parquet-converter.conversion-interval duration
+    	The frequency at which the conversion runs. (default 1m0s)
+  -parquet-converter.data-dir string
+    	Directory to temporarily store blocks during conversion. This directory is not required to persist between restarts. (default "./data-parquet-converter/")
+  -parquet-converter.disabled-tenants comma-separated-list-of-strings
+    	Comma-separated list of tenants that cannot have their TSDB blocks converted into Parquet. If specified, and the Parquet-converter would normally pick a given tenant to convert the blocks to Parquet (via -parquet-converter.enabled-tenants or sharding), it is ignored instead.
+  -parquet-converter.discovery-interval duration
+    	The frequency at which user and block discovery runs. (default 5m0s)
+  -parquet-converter.enabled-tenants comma-separated-list-of-strings
+    	Comma-separated list of tenants that can have their TSDB blocks converted into Parquet. If specified, the Parquet-converter only converts these tenants. Otherwise, it converts all tenants. Subject to sharding.
+  -parquet-converter.max-block-age duration
+    	Maximum age of blocks to convert. Blocks older than this will be skipped. Set to 0 to disable age filtering.
+  -parquet-converter.ring.consul.acl-token string
+    	ACL Token used to interact with Consul.
+  -parquet-converter.ring.consul.cas-retry-delay duration
+    	Maximum duration to wait before retrying a Compare And Swap (CAS) operation. (default 1s)
+  -parquet-converter.ring.consul.client-timeout duration
+    	HTTP timeout when talking to Consul (default 20s)
+  -parquet-converter.ring.consul.consistent-reads
+    	Enable consistent reads to Consul.
+  -parquet-converter.ring.consul.hostname string
+    	Hostname and port of Consul. (default "localhost:8500")
+  -parquet-converter.ring.consul.watch-burst-size int
+    	Burst size used in rate limit. Values less than 1 are treated as 1. (default 1)
+  -parquet-converter.ring.consul.watch-rate-limit float
+    	Rate limit when watching key or prefix in Consul, in requests per second. 0 disables the rate limit. (default 1)
+  -parquet-converter.ring.etcd.dial-timeout duration
+    	The dial timeout for the etcd connection. (default 10s)
+  -parquet-converter.ring.etcd.endpoints string
+    	The etcd endpoints to connect to.
+  -parquet-converter.ring.etcd.max-retries int
+    	The maximum number of retries to do for failed ops. (default 10)
+  -parquet-converter.ring.etcd.password string
+    	Etcd password.
+  -parquet-converter.ring.etcd.tls-ca-path string
+    	Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
+  -parquet-converter.ring.etcd.tls-cert-path string
+    	Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
+  -parquet-converter.ring.etcd.tls-cipher-suites string
+    	Override the default cipher suite list (separated by commas).
+  -parquet-converter.ring.etcd.tls-enabled
+    	Enable TLS.
+  -parquet-converter.ring.etcd.tls-insecure-skip-verify
+    	Skip validating server certificate.
+  -parquet-converter.ring.etcd.tls-key-path string
+    	Path to the key for the client certificate. Also requires the client certificate to be configured.
+  -parquet-converter.ring.etcd.tls-min-version string
+    	Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13
+  -parquet-converter.ring.etcd.tls-server-name string
+    	Override the expected name on the server certificate.
+  -parquet-converter.ring.etcd.username string
+    	Etcd username.
+  -parquet-converter.ring.heartbeat-period duration
+    	Period at which to heartbeat to the ring. 0 = disabled. (default 15s)
+  -parquet-converter.ring.heartbeat-timeout duration
+    	The heartbeat timeout after which parquet-converters are considered unhealthy within the ring. 0 = never (timeout disabled). (default 1m0s)
+  -parquet-converter.ring.instance-addr string
+    	IP address to advertise in the ring. Default is auto-detected.
+  -parquet-converter.ring.instance-enable-ipv6
+    	Enable using a IPv6 instance address. (default false)
+  -parquet-converter.ring.instance-id string
+    	Instance ID to register in the ring. (default "<hostname>")
+  -parquet-converter.ring.instance-interface-names string
+    	List of network interface names to look up when finding the instance IP address. (default [<private network interfaces>])
+  -parquet-converter.ring.instance-port int
+    	Port to advertise in the ring (defaults to -server.grpc-listen-port).
+  -parquet-converter.ring.multi.mirror-enabled
+    	Mirror writes to secondary store.
+  -parquet-converter.ring.multi.mirror-timeout duration
+    	Timeout for storing value to secondary store. (default 2s)
+  -parquet-converter.ring.multi.primary string
+    	Primary backend storage used by multi-client.
+  -parquet-converter.ring.multi.secondary string
+    	Secondary backend storage used by multi-client.
+  -parquet-converter.ring.prefix string
+    	The prefix for the keys in the store. Should end with a /. (default "parquet-converters/")
+  -parquet-converter.ring.store string
+    	Backend storage to use for the ring. Supported values are: consul, etcd, inmemory, memberlist, multi. (default "memberlist")
+  -parquet-converter.ring.wait-active-instance-timeout duration
+    	Timeout for waiting on Parquet-converter to become ACTIVE in the ring. (default 10m0s)
+  -parquet-converter.ring.wait-stability-max-duration duration
+    	Maximum time to wait for ring stability at startup. If the Parquet-converter ring keeps changing after this period of time, the Parquet-converter starts anyway. (default 5m0s)
+  -parquet-converter.ring.wait-stability-min-duration duration
+    	Minimum time to wait for ring stability at startup. Set to 0 to disable.
   -print.config
     	Print the config and exit.
   -querier.active-series-results-max-size-bytes int
@@ -3109,6 +3199,8 @@ Usage of ./cmd/mimir/mimir:
     	[experimental] Multiple of the default replication factor that should be used for recent blocks. Minimum value is 2 (default 2)
   -store-gateway.enabled-tenants comma-separated-list-of-strings
     	Comma separated list of tenants that can be loaded by the store-gateway. If specified, only blocks for these tenants will be loaded by the store-gateway, otherwise all tenants can be loaded. Subject to sharding.
+  -store-gateway.parquet-enabled
+    	[experimental] Whether to query Parquet files for block instead of the native Prometheus TSDB files.
   -store-gateway.sharding-ring.auto-forget-enabled
     	[deprecated] When enabled, a store-gateway is automatically removed from the ring after failing to heartbeat the ring for a period longer than the configured -store-gateway.sharding-ring.auto-forget-unhealthy-periods times the configured -store-gateway.sharding-ring.heartbeat-timeout. This setting is deprecated. Set -store-gateway.sharding-ring.auto-forget-unhealthy-periods to 0 to disable auto-forget. (default true)
   -store-gateway.sharding-ring.auto-forget-unhealthy-periods int
 
@@ -543,6 +543,26 @@ Usage of ./cmd/mimir/mimir:
     	List of network interface names to look up when finding the instance IP address. (default [<private network interfaces>])
   -overrides-exporter.ring.store string
     	Backend storage to use for the ring. Supported values are: consul, etcd, inmemory, memberlist, multi. (default "memberlist")
+  -parquet-converter.conversion-interval duration
+    	The frequency at which the conversion runs. (default 1m0s)
+  -parquet-converter.data-dir string
+    	Directory to temporarily store blocks during conversion. This directory is not required to persist between restarts. (default "./data-parquet-converter/")
+  -parquet-converter.discovery-interval duration
+    	The frequency at which user and block discovery runs. (default 5m0s)
+  -parquet-converter.max-block-age duration
+    	Maximum age of blocks to convert. Blocks older than this will be skipped. Set to 0 to disable age filtering.
+  -parquet-converter.ring.consul.hostname string
+    	Hostname and port of Consul. (default "localhost:8500")
+  -parquet-converter.ring.etcd.endpoints string
+    	The etcd endpoints to connect to.
+  -parquet-converter.ring.etcd.password string
+    	Etcd password.
+  -parquet-converter.ring.etcd.username string
+    	Etcd username.
+  -parquet-converter.ring.instance-interface-names string
+    	List of network interface names to look up when finding the instance IP address. (default [<private network interfaces>])
+  -parquet-converter.ring.store string
+    	Backend storage to use for the ring. Supported values are: consul, etcd, inmemory, memberlist, multi. (default "memberlist")
   -print.config
     	Print the config and exit.
   -querier.cardinality-analysis-enabled
 
@@ -35,6 +35,9 @@ std.manifestYamlDoc({
 
     // If true, a query-tee instance with a single backend is started.
     enable_query_tee: false,
+
+    // If true, start parquet-converter
+    enable_parquet: false,
   },
 
   // We explicitely list all important services here, so that it's easy to disable them by commenting out.
@@ -44,6 +47,7 @@ std.manifestYamlDoc({
     self.read_components +  // Querier, Frontend and query-scheduler, if enabled.
     self.store_gateways(3) +
     self.compactor +
+    (if $._config.enable_parquet then self.parquet_converter else {}) +
     self.rulers(2) +
     self.alertmanagers(3) +
     self.nginx +
@@ -142,6 +146,14 @@ std.manifestYamlDoc({
     }),
   },
 
+  parquet_converter:: {
+    'parquet-converter': mimirService({
+      name: 'parquet-converter',
+      target: 'parquet-converter',
+      httpPort: 8040,
+    }),
+  },
+
   rulers(count):: if count <= 0 then {} else {
     ['ruler-%d' % id]: mimirService({
       name: 'ruler-' + id,
@@ -170,6 +182,8 @@ std.manifestYamlDoc({
       target: 'store-gateway',
       httpPort: 8010 + id,
       jaegerApp: 'store-gateway-%d' % id,
+      extraArguments:
+        if $._config.enable_parquet then '-store-gateway.parquet-enabled=true' else '',
     })
     for id in std.range(1, count)
   },
 
@@ -453,6 +453,124 @@ overrides_exporter:
   # CLI flag: -overrides-exporter.enabled-metrics
   [enabled_metrics: <string> | default = "ingestion_rate,ingestion_burst_size,max_global_series_per_user,max_global_series_per_metric,max_global_exemplars_per_user,max_fetched_chunks_per_query,max_fetched_series_per_query,max_fetched_chunk_bytes_per_query,ruler_max_rules_per_rule_group,ruler_max_rule_groups_per_tenant"]
 
+parquet_converter:
+  # (advanced) Comma-separated list of tenants that can have their TSDB blocks
+  # converted into Parquet. If specified, the Parquet-converter only converts
+  # these tenants. Otherwise, it converts all tenants. Subject to sharding.
+  # CLI flag: -parquet-converter.enabled-tenants
+  [enabled_tenants: <string> | default = ""]
+
+  # (advanced) Comma-separated list of tenants that cannot have their TSDB
+  # blocks converted into Parquet. If specified, and the Parquet-converter would
+  # normally pick a given tenant to convert the blocks to Parquet (via
+  # -parquet-converter.enabled-tenants or sharding), it is ignored instead.
+  # CLI flag: -parquet-converter.disabled-tenants
+  [disabled_tenants: <string> | default = ""]
+
+  # Directory to temporarily store blocks during conversion. This directory is
+  # not required to persist between restarts.
+  # CLI flag: -parquet-converter.data-dir
+  [data_dir: <string> | default = "./data-parquet-converter/"]
+
+  # The frequency at which the conversion runs.
+  # CLI flag: -parquet-converter.conversion-interval
+  [conversion_interval: <duration> | default = 1m]
+
+  # The frequency at which user and block discovery runs.
+  # CLI flag: -parquet-converter.discovery-interval
+  [discovery_interval: <duration> | default = 5m]
+
+  # Maximum age of blocks to convert. Blocks older than this will be skipped.
+  # Set to 0 to disable age filtering.
+  # CLI flag: -parquet-converter.max-block-age
+  [max_block_age: <duration> | default = 0s]
+
+  sharding_ring:
+    # The key-value store used to share the hash ring across multiple instances.
+    kvstore:
+      # Backend storage to use for the ring. Supported values are: consul, etcd,
+      # inmemory, memberlist, multi.
+      # CLI flag: -parquet-converter.ring.store
+      [store: <string> | default = "memberlist"]
+
+      # (advanced) The prefix for the keys in the store. Should end with a /.
+      # CLI flag: -parquet-converter.ring.prefix
+      [prefix: <string> | default = "parquet-converters/"]
+
+      # The consul block configures the consul client.
+      # The CLI flags prefix for this block configuration is:
+      # parquet-converter.ring
+      [consul: <consul>]
+
+      # The etcd block configures the etcd client.
+      # The CLI flags prefix for this block configuration is:
+      # parquet-converter.ring
+      [etcd: <etcd>]
+
+      multi:
+        # (advanced) Primary backend storage used by multi-client.
+        # CLI flag: -parquet-converter.ring.multi.primary
+        [primary: <string> | default = ""]
+
+        # (advanced) Secondary backend storage used by multi-client.
+        # CLI flag: -parquet-converter.ring.multi.secondary
+        [secondary: <string> | default = ""]
+
+        # (advanced) Mirror writes to secondary store.
+        # CLI flag: -parquet-converter.ring.multi.mirror-enabled
+        [mirror_enabled: <boolean> | default = false]
+
+        # (advanced) Timeout for storing value to secondary store.
+        # CLI flag: -parquet-converter.ring.multi.mirror-timeout
+        [mirror_timeout: <duration> | default = 2s]
+
+    # (advanced) Period at which to heartbeat to the ring. 0 = disabled.
+    # CLI flag: -parquet-converter.ring.heartbeat-period
+    [heartbeat_period: <duration> | default = 15s]
+
+    # (advanced) The heartbeat timeout after which parquet-converters are
+    # considered unhealthy within the ring. 0 = never (timeout disabled).
+    # CLI flag: -parquet-converter.ring.heartbeat-timeout
+    [heartbeat_timeout: <duration> | default = 1m]
+
+    # (advanced) Instance ID to register in the ring.
+    # CLI flag: -parquet-converter.ring.instance-id
+    [instance_id: <string> | default = "<hostname>"]
+
+    # List of network interface names to look up when finding the instance IP
+    # address.
+    # CLI flag: -parquet-converter.ring.instance-interface-names
+    [instance_interface_names: <list of strings> | default = [<private network interfaces>]]
+
+    # (advanced) Port to advertise in the ring (defaults to
+    # -server.grpc-listen-port).
+    # CLI flag: -parquet-converter.ring.instance-port
+    [instance_port: <int> | default = 0]
+
+    # (advanced) IP address to advertise in the ring. Default is auto-detected.
+    # CLI flag: -parquet-converter.ring.instance-addr
+    [instance_addr: <string> | default = ""]
+
+    # (advanced) Enable using a IPv6 instance address. (default false)
+    # CLI flag: -parquet-converter.ring.instance-enable-ipv6
+    [instance_enable_ipv6: <boolean> | default = false]
+
+    # (advanced) Minimum time to wait for ring stability at startup. Set to 0 to
+    # disable.
+    # CLI flag: -parquet-converter.ring.wait-stability-min-duration
+    [wait_stability_min_duration: <duration> | default = 0s]
+
+    # (advanced) Maximum time to wait for ring stability at startup. If the
+    # Parquet-converter ring keeps changing after this period of time, the
+    # Parquet-converter starts anyway.
+    # CLI flag: -parquet-converter.ring.wait-stability-max-duration
+    [wait_stability_max_duration: <duration> | default = 5m]
+
+    # (advanced) Timeout for waiting on Parquet-converter to become ACTIVE in
+    # the ring.
+    # CLI flag: -parquet-converter.ring.wait-active-instance-timeout
+    [wait_active_instance_timeout: <duration> | default = 10m]
+
 # The common block holds configurations that configure multiple components at a
 # time.
 [common: <common>]
@@ -2931,6 +3049,7 @@ The `etcd` block configures the etcd client. The supported CLI flags `<prefix>`
 - `ingester.partition-ring`
 - `ingester.ring`
 - `overrides-exporter.ring`
+- `parquet-converter.ring`
 - `query-scheduler.ring`
 - `ruler.ring`
 - `store-gateway.sharding-ring`
@@ -3036,6 +3155,7 @@ The `consul` block configures the consul client. The supported CLI flags `<prefi
 - `ingester.partition-ring`
 - `ingester.ring`
 - `overrides-exporter.ring`
+- `parquet-converter.ring`
 - `query-scheduler.ring`
 - `ruler.ring`
 - `store-gateway.sharding-ring`
@@ -4774,6 +4894,22 @@ bucket_store:
   # CLI flag: -blocks-storage.bucket-store.series-fetch-preference
   [series_fetch_preference: <float> | default = 0.75]
 
+  # (advanced) Maximum number of rows in a parquet file. If the number of rows
+  # exceeds this value the query will stop with limit error.
+  # CLI flag: -blocks-storage.bucket-store.parquet-max-row-count
+  [parquet_max_row_count: <int> | default = 0]
+
+  # (advanced) Maximum size in bytes that can be fetched from the parquet chunks
+  # file. If the size exceeds this value the query will stop with limit error.
+  # CLI flag: -blocks-storage.bucket-store.parquet-max-chunk-size-bytes
+  [parquet_max_chunk_size_bytes: <int> | default = 0]
+
+  # (advanced) Maximum size in bytes that can be fetched from the parquet labels
+  # and chunks data files combined in a single query. If the size exceeds this
+  # value the query will stop with limit error.
+  # CLI flag: -blocks-storage.bucket-store.parquet-max-data-size-bytes
+  [parquet_max_data_size_bytes: <int> | default = 0]
+
 tsdb:
   # Directory to store TSDBs (including WAL) in the ingesters. This directory is
   # required to be persisted between restarts.
@@ -5369,6 +5505,11 @@ dynamic_replication:
 # ignored instead.
 # CLI flag: -store-gateway.disabled-tenants
 [disabled_tenants: <string> | default = ""]
+
+# (experimental) Whether to query Parquet files for block instead of the native
+# Prometheus TSDB files.
+# CLI flag: -store-gateway.parquet-enabled
+[parquet_enabled: <boolean> | default = false]
 ```
 
 ### memcached