Skip to content

Commit 2161a99

Browse files
francoposajesusvazquez
authored andcommitted
parquet store-gateways: open labels file from bucket with flag (#12091)
<!-- Thanks for sending a pull request! Before submitting: 1. Read our CONTRIBUTING.md guide 2. Rebase your PR if it gets out of sync with main --> #### What this PR does ``` -blocks-storage.bucket-store.parquet-load-index-to-disk True to download the Parquet labels file to disk before opening it. False to open it directly from the bucket. (default true) ``` #### Which issue(s) this PR fixes or relates to Fixes #<issue number> #### Checklist - [ ] Tests updated. - [ ] Documentation added. - [ ] `CHANGELOG.md` updated - the order of entries should be `[CHANGE]`, `[FEATURE]`, `[ENHANCEMENT]`, `[BUGFIX]`. If changelog entry is not needed, please add the `changelog-not-needed` label to the PR. - [ ] [`about-versioning.md`](https://github.com/grafana/mimir/blob/main/docs/sources/mimir/configure/about-versioning.md) updated with experimental features.
1 parent 83032b2 commit 2161a99

File tree

11 files changed

+651
-426
lines changed

11 files changed

+651
-426
lines changed

cmd/mimir/config-descriptor.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10173,6 +10173,17 @@
1017310173
"fieldType": "float",
1017410174
"fieldCategory": "advanced"
1017510175
},
10176+
{
10177+
"kind": "field",
10178+
"name": "parquet_load_index_to_disk",
10179+
"required": false,
10180+
"desc": "True to download the Parquet labels file to disk before opening it. False to open it directly from the bucket.",
10181+
"fieldValue": null,
10182+
"fieldDefaultValue": true,
10183+
"fieldFlag": "blocks-storage.bucket-store.parquet-load-index-to-disk",
10184+
"fieldType": "boolean",
10185+
"fieldCategory": "advanced"
10186+
},
1017610187
{
1017710188
"kind": "field",
1017810189
"name": "parquet_max_row_count",

cmd/mimir/help-all.txt.tmpl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,8 @@ Usage of ./cmd/mimir/mimir:
609609
How long to cache list of blocks for each tenant. (default 5m0s)
610610
-blocks-storage.bucket-store.metadata-cache.tenants-list-ttl duration
611611
How long to cache list of tenants in the bucket. (default 15m0s)
612+
-blocks-storage.bucket-store.parquet-load-index-to-disk
613+
True to download the Parquet labels file to disk before opening it. False to open it directly from the bucket. (default true)
612614
-blocks-storage.bucket-store.parquet-max-chunk-size-bytes uint
613615
Maximum size in bytes that can be fetched from the parquet chunks file. If the size exceeds this value the query will stop with limit error.
614616
-blocks-storage.bucket-store.parquet-max-data-size-bytes uint

development/mimir-microservices-mode/docker-compose.jsonnet

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ std.manifestYamlDoc({
3838

3939
// If true, start parquet-converter
4040
enable_parquet: false,
41+
// If true, emulate classic store-gateway behavior by loading the Parquet labels file to disk
42+
parquet_store_gateway_load_index_to_disk: false,
4143
},
4244

4345
// We explicitely list all important services here, so that it's easy to disable them by commenting out.
@@ -176,14 +178,20 @@ std.manifestYamlDoc({
176178
for id in std.range(1, count)
177179
},
178180

181+
local extraStoreGatewayArgs = std.join(
182+
' ', [
183+
(if $._config.enable_parquet then '-log.level=debug -store-gateway.parquet-enabled=true' else null),
184+
(if ($._config.enable_parquet && !$._config.parquet_store_gateway_load_index_to_disk) then '-blocks-storage.bucket-store.parquet-load-index-to-disk=false' else null),
185+
]
186+
),
187+
179188
store_gateways(count):: {
180189
['store-gateway-%d' % id]: mimirService({
181190
name: 'store-gateway-' + id,
182191
target: 'store-gateway',
183192
httpPort: 8010 + id,
184193
jaegerApp: 'store-gateway-%d' % id,
185-
extraArguments:
186-
if $._config.enable_parquet then '-store-gateway.parquet-enabled=true' else '',
194+
extraArguments: extraStoreGatewayArgs,
187195
})
188196
for id in std.range(1, count)
189197
},

docs/sources/mimir/configure/configuration-parameters/index.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4894,6 +4894,11 @@ bucket_store:
48944894
# CLI flag: -blocks-storage.bucket-store.series-fetch-preference
48954895
[series_fetch_preference: <float> | default = 0.75]
48964896
4897+
# (advanced) True to download the Parquet labels file to disk before opening
4898+
# it. False to open it directly from the bucket.
4899+
# CLI flag: -blocks-storage.bucket-store.parquet-load-index-to-disk
4900+
[parquet_load_index_to_disk: <boolean> | default = true]
4901+
48974902
# (advanced) Maximum number of rows in a parquet file. If the number of rows
48984903
# exceeds this value the query will stop with limit error.
48994904
# CLI flag: -blocks-storage.bucket-store.parquet-max-row-count

0 commit comments

Comments
 (0)