Skip to content

Commit fe248ab

Browse files
francoposanpazosmendez
authored andcommitted
parquet store-gateways: open labels file from bucket with flag (#12091)
<!-- Thanks for sending a pull request! Before submitting: 1. Read our CONTRIBUTING.md guide 2. Rebase your PR if it gets out of sync with main --> #### What this PR does ``` -blocks-storage.bucket-store.parquet-load-index-to-disk True to download the Parquet labels file to disk before opening it. False to open it directly from the bucket. (default true) ``` #### Which issue(s) this PR fixes or relates to Fixes #<issue number> #### Checklist - [ ] Tests updated. - [ ] Documentation added. - [ ] `CHANGELOG.md` updated - the order of entries should be `[CHANGE]`, `[FEATURE]`, `[ENHANCEMENT]`, `[BUGFIX]`. If changelog entry is not needed, please add the `changelog-not-needed` label to the PR. - [ ] [`about-versioning.md`](https://github.com/grafana/mimir/blob/main/docs/sources/mimir/configure/about-versioning.md) updated with experimental features.
1 parent 988774b commit fe248ab

File tree

11 files changed

+651
-426
lines changed

11 files changed

+651
-426
lines changed

cmd/mimir/config-descriptor.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11379,6 +11379,17 @@
1137911379
"fieldType": "float",
1138011380
"fieldCategory": "advanced"
1138111381
},
11382+
{
11383+
"kind": "field",
11384+
"name": "parquet_load_index_to_disk",
11385+
"required": false,
11386+
"desc": "True to download the Parquet labels file to disk before opening it. False to open it directly from the bucket.",
11387+
"fieldValue": null,
11388+
"fieldDefaultValue": true,
11389+
"fieldFlag": "blocks-storage.bucket-store.parquet-load-index-to-disk",
11390+
"fieldType": "boolean",
11391+
"fieldCategory": "advanced"
11392+
},
1138211393
{
1138311394
"kind": "field",
1138411395
"name": "parquet_max_row_count",

cmd/mimir/help-all.txt.tmpl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -771,6 +771,8 @@ Usage of ./cmd/mimir/mimir:
771771
How long to cache list of blocks for each tenant. (default 5m0s)
772772
-blocks-storage.bucket-store.metadata-cache.tenants-list-ttl duration
773773
How long to cache list of tenants in the bucket. (default 15m0s)
774+
-blocks-storage.bucket-store.parquet-load-index-to-disk
775+
True to download the Parquet labels file to disk before opening it. False to open it directly from the bucket. (default true)
774776
-blocks-storage.bucket-store.parquet-max-chunk-size-bytes uint
775777
Maximum size in bytes that can be fetched from the parquet chunks file. If the size exceeds this value the query will stop with limit error.
776778
-blocks-storage.bucket-store.parquet-max-data-size-bytes uint

development/mimir-microservices-mode/docker-compose.jsonnet

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ std.manifestYamlDoc({
4141

4242
// If true, start parquet-converter
4343
enable_parquet: false,
44+
// If true, emulate classic store-gateway behavior by loading the Parquet labels file to disk
45+
parquet_store_gateway_load_index_to_disk: false,
4446
},
4547

4648
// We explicitely list all important services here, so that it's easy to disable them by commenting out.
@@ -179,14 +181,20 @@ std.manifestYamlDoc({
179181
for id in std.range(1, count)
180182
},
181183

184+
local extraStoreGatewayArgs = std.join(
185+
' ', [
186+
(if $._config.enable_parquet then '-log.level=debug -store-gateway.parquet-enabled=true' else null),
187+
(if ($._config.enable_parquet && !$._config.parquet_store_gateway_load_index_to_disk) then '-blocks-storage.bucket-store.parquet-load-index-to-disk=false' else null),
188+
]
189+
),
190+
182191
store_gateways(count):: {
183192
['store-gateway-%d' % id]: mimirService({
184193
name: 'store-gateway-' + id,
185194
target: 'store-gateway',
186195
httpPort: 8010 + id,
187196
jaegerApp: 'store-gateway-%d' % id,
188-
extraArguments:
189-
if $._config.enable_parquet then '-store-gateway.parquet-enabled=true' else '',
197+
extraArguments: extraStoreGatewayArgs,
190198
})
191199
for id in std.range(1, count)
192200
},

docs/sources/mimir/configure/configuration-parameters/index.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4906,6 +4906,11 @@ bucket_store:
49064906
# CLI flag: -blocks-storage.bucket-store.series-fetch-preference
49074907
[series_fetch_preference: <float> | default = 0.75]
49084908
4909+
# (advanced) True to download the Parquet labels file to disk before opening
4910+
# it. False to open it directly from the bucket.
4911+
# CLI flag: -blocks-storage.bucket-store.parquet-load-index-to-disk
4912+
[parquet_load_index_to_disk: <boolean> | default = true]
4913+
49094914
# (advanced) Maximum number of rows in a parquet file. If the number of rows
49104915
# exceeds this value the query will stop with limit error.
49114916
# CLI flag: -blocks-storage.bucket-store.parquet-max-row-count

0 commit comments

Comments
 (0)