Skip to content

Change duckdb.max_memory/duckdb.memory_limit to integer GUC #883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 15, 2025
Merged

Change duckdb.max_memory/duckdb.memory_limit to integer GUC #883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2dcd941
use integer for memory guc
Aug 8, 2025
b221113
fix regress
Aug 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion include/pgduckdb/pgduckdb_guc.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ extern bool duckdb_unsafe_allow_mixed_transactions;
extern bool duckdb_convert_unsupported_numeric_to_double;
extern bool duckdb_log_pg_explain;
extern int duckdb_maximum_threads;
extern char *duckdb_maximum_memory;
extern int duckdb_maximum_memory;
extern char *duckdb_disabled_filesystems;
extern bool duckdb_enable_external_access;
extern bool duckdb_allow_community_extensions;
Expand Down
10 changes: 7 additions & 3 deletions src/pgduckdb_duckdb.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,13 @@ DuckDBManager::Initialize() {
SET_DUCKDB_OPTION(temporary_directory);
SET_DUCKDB_OPTION(extension_directory);

if (duckdb_maximum_memory != NULL && strlen(duckdb_maximum_memory) != 0) {
config.options.maximum_memory = duckdb::DBConfig::ParseMemoryLimit(duckdb_maximum_memory);
elog(DEBUG2, "[PGDuckDB] Set DuckDB option: 'maximum_memory'=%s", duckdb_maximum_memory);
if (duckdb_maximum_memory > 0) {
// Convert the memory limit from MB (as set by Postgres GUC_UNIT_MB, which is actually MiB; see
// memory_unit_conversion_table in guc.c) to a string with the "MiB" suffix, as required by DuckDB's memory
// parser. This ensures the value is interpreted correctly by DuckDB.
std::string memory_limit = std::to_string(duckdb_maximum_memory) + "MiB";
config.options.maximum_memory = duckdb::DBConfig::ParseMemoryLimit(memory_limit);
elog(DEBUG2, "[PGDuckDB] Set DuckDB option: 'maximum_memory'=%dMB", duckdb_maximum_memory);
}
if (duckdb_max_temp_directory_size != NULL && strlen(duckdb_max_temp_directory_size) != 0) {
config.SetOptionByName("max_temp_directory_size", duckdb_max_temp_directory_size);
Expand Down
14 changes: 8 additions & 6 deletions src/pgduckdb_guc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include <type_traits>
#include <climits>

#include "pgduckdb/pgduckdb_duckdb.hpp"
#include "pgduckdb/pgduckdb_guc.hpp"
Expand Down Expand Up @@ -126,7 +127,7 @@ char *duckdb_postgres_role = strdup("");
bool duckdb_force_motherduck_views = false;

int duckdb_maximum_threads = -1;
char *duckdb_maximum_memory = strdup("4GB");
int duckdb_maximum_memory = 4096; /* 4GB in MB */
char *duckdb_disabled_filesystems = strdup("");
bool duckdb_enable_external_access = true;
bool duckdb_allow_community_extensions = false;
Expand Down Expand Up @@ -191,11 +192,12 @@ InitGUC() {
"Whether known extensions are allowed to be automatically loaded when a DuckDB query depends on them",
&duckdb_autoload_known_extensions, PGC_SUSET);

DefineCustomDuckDBVariable("duckdb.max_memory", "The maximum memory DuckDB can use (e.g., 1GB)",
&duckdb_maximum_memory, PGC_SUSET);
DefineCustomDuckDBVariable("duckdb.memory_limit",
"The maximum memory DuckDB can use (e.g., 1GB), alias for duckdb.max_memory",
&duckdb_maximum_memory, PGC_SUSET);
DefineCustomDuckDBVariable("duckdb.max_memory", "The maximum memory DuckDB can use in MB (e.g., 4096 for 4GB)",
&duckdb_maximum_memory, 0, INT_MAX, PGC_SUSET, GUC_UNIT_MB);
DefineCustomDuckDBVariable(
"duckdb.memory_limit",
"The maximum memory DuckDB can use in MB (e.g., 4096 for 4GB), alias for duckdb.max_memory",
&duckdb_maximum_memory, 0, INT_MAX, PGC_SUSET, GUC_UNIT_MB);

DefineCustomDuckDBVariable(
"duckdb.temporary_directory",
Expand Down
20 changes: 6 additions & 14 deletions test/regression/expected/gucs.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,22 @@ show duckdb.memory_limit;
4GB
(1 row)

select * from duckdb.query($$ select current_setting('memory_limit') == '3.7 GiB' $$);
(current_setting('memory_limit') = '3.7 GiB')
-----------------------------------------------
t
select * from duckdb.query($$ select current_setting('memory_limit')$$);
current_setting('memory_limit')
---------------------------------
4.0 GiB
(1 row)

CALL duckdb.recycle_ddb();
set duckdb.memory_limit = '1GiB';
set duckdb.memory_limit = 1024;
select * from duckdb.query($$ select current_setting('memory_limit') $$);
current_setting('memory_limit')
---------------------------------
1.0 GiB
(1 row)

CALL duckdb.recycle_ddb();
set duckdb.memory_limit = '';
Copy link
Collaborator

@JelteF JelteF Aug 13, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we allow setting this to -1 or something to represent the "duckdb default"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Setting the value to 0 will use the 'DuckDB default'. It appears that DuckDB determines this default by checking the system spec. I removed this case since it may lead to unstable case.

select * from duckdb.query($$ select current_setting('memory_limit') != '3.7 GiB' $$);
(current_setting('memory_limit') != '3.7 GiB')
------------------------------------------------
t
(1 row)

-- Don't recycle DuckDB, make sure we get an error:
set duckdb.memory_limit = '1GiB';
set duckdb.memory_limit = '1GB';
ERROR: Cannot set this variable after DuckDB has been initialized. Reconnect to Postgres or use `duckdb.recycle_ddb()` to reset the DuckDB instance.
set duckdb.threads = 42;
ERROR: Cannot set this variable after DuckDB has been initialized. Reconnect to Postgres or use `duckdb.recycle_ddb()` to reset the DuckDB instance.
Expand Down
10 changes: 3 additions & 7 deletions test/regression/sql/gucs.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
show duckdb.memory_limit;
select * from duckdb.query($$ select current_setting('memory_limit') == '3.7 GiB' $$);
select * from duckdb.query($$ select current_setting('memory_limit')$$);
CALL duckdb.recycle_ddb();

set duckdb.memory_limit = '1GiB';
set duckdb.memory_limit = 1024;
select * from duckdb.query($$ select current_setting('memory_limit') $$);
CALL duckdb.recycle_ddb();

set duckdb.memory_limit = '';
select * from duckdb.query($$ select current_setting('memory_limit') != '3.7 GiB' $$);

-- Don't recycle DuckDB, make sure we get an error:
set duckdb.memory_limit = '1GiB';
set duckdb.memory_limit = '1GB';

set duckdb.threads = 42;

Expand Down