Skip to content

Commit 6537485

Browse files
bikramSingh91tanjialiang
authored andcommitted
[native] Add a metric to track expected reduction of memory and minor fixes
Summary: - Add a metric `presto_cpp.memory_pushback_expected_reduction_bytes` to track expected reduction in memory after a pushback attempt as this can be different from actual reduction due to other threads allocating simultaneously - Fix registeration of a pushback metric - Make periodicCb() API non-const to allow it to update internal state if required
1 parent 96b716d commit 6537485

File tree

5 files changed

+24
-6
lines changed

5 files changed

+24
-6
lines changed

presto-native-execution/presto_cpp/main/PeriodicMemoryChecker.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,8 @@ void PeriodicMemoryChecker::pushbackMemory() {
211211
kCounterMemoryPushbackLatencyMs, latencyUs / 1000);
212212
const auto actualFreedBytes = std::max<int64_t>(
213213
0, static_cast<int64_t>(currentMemBytes) - systemUsedMemoryBytes());
214+
RECORD_HISTOGRAM_METRIC_VALUE(
215+
kCounterMemoryPushbackExpectedReductionBytes, freedBytes);
214216
RECORD_HISTOGRAM_METRIC_VALUE(
215217
kCounterMemoryPushbackReductionBytes, actualFreedBytes);
216218
LOG(INFO) << "Memory pushback shrunk " << velox::succinctBytes(freedBytes)

presto-native-execution/presto_cpp/main/PeriodicMemoryChecker.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ class PeriodicMemoryChecker {
8787

8888
/// Callback function that is invoked by 'PeriodicMemoryChecker' periodically.
8989
/// Light operations such as stats reporting can be done in this call back.
90-
virtual void periodicCb() const = 0;
90+
virtual void periodicCb() = 0;
9191

9292
/// Callback function that performs a heap dump. Returns true if dump is
9393
/// successful.

presto-native-execution/presto_cpp/main/common/Counters.cpp

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,16 @@ void registerPrestoMetrics() {
103103
DEFINE_HISTOGRAM_METRIC(
104104
kCounterMemoryPushbackLatencyMs, 10'000, 0, 100'000, 50, 90, 99, 100);
105105
DEFINE_HISTOGRAM_METRIC(
106-
kCounterMemoryPushbackLatencyMs,
106+
kCounterMemoryPushbackReductionBytes,
107+
100l * 1024 * 1024, // 100MB
108+
0,
109+
15l * 1024 * 1024 * 1024, // 15GB
110+
50,
111+
90,
112+
99,
113+
100);
114+
DEFINE_HISTOGRAM_METRIC(
115+
kCounterMemoryPushbackExpectedReductionBytes,
107116
100l * 1024 * 1024, // 100MB
108117
0,
109118
15l * 1024 * 1024 * 1024, // 15GB

presto-native-execution/presto_cpp/main/common/Counters.h

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,9 +166,16 @@ constexpr folly::StringPiece kCounterMemoryPushbackCount{
166166
/// reports P50, P90, P99, and P100.
167167
constexpr folly::StringPiece kCounterMemoryPushbackLatencyMs{
168168
"presto_cpp.memory_pushback_latency_ms"};
169-
/// Distribution of reduction in memory usage achieved by each memory pushback
170-
/// attempt. This is to gauge its effectiveness. In range of [0, 15GB] with 150
171-
/// buckets and reports P50, P90, P99, and P100.
169+
/// Distribution of actual reduction in memory usage achieved by each memory
170+
/// pushback attempt. This is to gauge its effectiveness. In range of [0, 15GB]
171+
/// with 150 buckets and reports P50, P90, P99, and P100.
172172
constexpr folly::StringPiece kCounterMemoryPushbackReductionBytes{
173173
"presto_cpp.memory_pushback_reduction_bytes"};
174+
/// Distribution of expected reduction in memory usage achieved by each memory
175+
/// pushback attempt. This is to gauge its effectiveness. In range of [0, 15GB]
176+
/// with 150 buckets and reports P50, P90, P99, and P100. The expected reduction
177+
/// can be different as other threads might have allocated memory in the
178+
/// meantime.
179+
constexpr folly::StringPiece kCounterMemoryPushbackExpectedReductionBytes{
180+
"presto_cpp.memory_pushback_expected_reduction_bytes"};
174181
} // namespace facebook::presto

presto-native-execution/presto_cpp/main/tests/PeriodicMemoryCheckerTest.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class PeriodicMemoryCheckerTest : public testing::Test {
5454
return mallocBytes_;
5555
}
5656

57-
void periodicCb() const override {
57+
void periodicCb() override {
5858
if (periodicCb_) {
5959
periodicCb_();
6060
}

0 commit comments

Comments
 (0)