Skip to content
This repository was archived by the owner on Oct 4, 2019. It is now read-only.

Commit 0bb80f9

Browse files
kotbegemotkotbegemot
authored
Merge pull request #502 from GolosChain/501-autoscale-shared-memory
Autoscale shared memory. #501
2 parents a556538 + ede2213 commit 0bb80f9

File tree

9 files changed

+146
-79
lines changed

9 files changed

+146
-79
lines changed

‎libraries/chain/database.cpp

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -168,15 +168,13 @@ namespace golos {
168168
while (itr.first.block_num() != last_block_num) {
169169
auto cur_block_num = itr.first.block_num();
170170
if (cur_block_num % 100000 == 0) {
171-
std::cerr << " " << double(cur_block_num * 100) /
172-
last_block_num << "% "
173-
<< cur_block_num << " of "
174-
<< last_block_num <<
175-
" ("
176-
<< (get_free_memory() / (1024 * 1024))
177-
<< "M free)\n";
171+
std::cerr
172+
<< " " << double(cur_block_num * 100) / last_block_num << "% "
173+
<< cur_block_num << " of " << last_block_num
174+
<< " (" << (free_memory() / (1024 * 1024)) << "M free)\n";
178175
}
179176
apply_block(itr.first, skip_flags);
177+
check_free_memory(true, itr.first.block_num());
180178
itr = _block_log.read_block(itr.second);
181179
}
182180

@@ -196,6 +194,56 @@ namespace golos {
196194

197195
}
198196

197+
void database::min_free_shared_memory_size(size_t value) {
198+
_min_free_shared_memory_size = value;
199+
}
200+
201+
void database::inc_shared_memory_size(size_t value) {
202+
_inc_shared_memory_size = value;
203+
}
204+
205+
void database::block_num_check_free_size(uint32_t value) {
206+
_block_num_check_free_memory = value;
207+
}
208+
209+
void database::check_free_memory(bool skip_print, uint32_t current_block_num) {
210+
if (0 != current_block_num % _block_num_check_free_memory) {
211+
return;
212+
}
213+
214+
uint64_t free_mem = free_memory();
215+
uint64_t max_mem = max_memory();
216+
217+
if (_inc_shared_memory_size != 0 && _min_free_shared_memory_size != 0 &&
218+
free_mem < _min_free_shared_memory_size
219+
) {
220+
size_t new_max = max_mem + _inc_shared_memory_size;
221+
wlog(
222+
"Memory is almost full on block ${block}, increasing to ${mem}M",
223+
("block", current_block_num)("mem", new_max / (1024 * 1024)));
224+
resize(new_max);
225+
free_mem = free_memory();
226+
uint32_t free_mb = uint32_t(free_mem / (1024 * 1024));
227+
wlog("Free memory is now ${free}M", ("free", free_mb));
228+
_last_free_gb_printed = free_mb / 1024;
229+
} else if (!skip_print && _inc_shared_memory_size == 0 && _min_free_shared_memory_size == 0) {
230+
uint32_t free_gb = uint32_t(free_mem / (1024 * 1024 * 1024));
231+
if ((free_gb < _last_free_gb_printed) || (free_gb > _last_free_gb_printed + 1)) {
232+
ilog(
233+
"Free memory is now ${n}G. Current block number: ${block}",
234+
("n", free_gb)("block", current_block_num));
235+
_last_free_gb_printed = free_gb;
236+
}
237+
238+
if (free_gb == 0) {
239+
uint32_t free_mb = uint32_t(free_mem / (1024 * 1024));
240+
if (free_mb <= 500 && current_block_num % 10 == 0) {
241+
elog("Free memory is now ${n}M. Increase shared file size immediately!", ("n", free_mb));
242+
}
243+
}
244+
}
245+
}
246+
199247
void database::wipe(const fc::path &data_dir, const fc::path &shared_mem_dir, bool include_blocks) {
200248
close();
201249
chainbase::database::wipe(shared_mem_dir);
@@ -637,6 +685,8 @@ namespace golos {
637685
FC_CAPTURE_AND_RETHROW((new_block))
638686
});
639687
});
688+
689+
check_free_memory(false, new_block.block_num());
640690
});
641691

642692
//fc::time_point end_time = fc::time_point::now();
@@ -689,7 +739,7 @@ namespace golos {
689739
// ilog( "pushing blocks from fork ${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->data.id()) );
690740
optional<fc::exception> except;
691741
try {
692-
auto session = start_undo_session(true);
742+
auto session = start_undo_session();
693743
apply_block((*ritr)->data, skip);
694744
session.push();
695745
}
@@ -715,7 +765,7 @@ namespace golos {
715765
for (auto ritr = branches.second.rbegin();
716766
ritr !=
717767
branches.second.rend(); ++ritr) {
718-
auto session = start_undo_session(true);
768+
auto session = start_undo_session();
719769
apply_block((*ritr)->data, skip);
720770
session.push();
721771
}
@@ -730,7 +780,7 @@ namespace golos {
730780
}
731781

732782
try {
733-
auto session = start_undo_session(true);
783+
auto session = start_undo_session();
734784
apply_block(new_block, skip);
735785
session.push();
736786
}
@@ -771,15 +821,15 @@ namespace golos {
771821
// If this is the first transaction pushed after applying a block, start a new undo session.
772822
// This allows us to quickly rewind to the clean state of the head block, in case a new block arrives.
773823
if (!_pending_tx_session.valid()) {
774-
_pending_tx_session = start_undo_session(true);
824+
_pending_tx_session = start_undo_session();
775825
}
776826

777827
// Create a temporary undo session as a child of _pending_tx_session.
778828
// The temporary session will be discarded by the destructor if
779829
// _apply_transaction fails. If we make it to merge(), we
780830
// apply the changes.
781831

782-
auto temp_session = start_undo_session(true);
832+
auto temp_session = start_undo_session();
783833
_apply_transaction(trx);
784834
_pending_tx.push_back(trx);
785835

@@ -843,7 +893,7 @@ namespace golos {
843893
// re-apply pending transactions in this method.
844894
//
845895
_pending_tx_session.reset();
846-
_pending_tx_session = start_undo_session(true);
896+
_pending_tx_session = start_undo_session();
847897

848898
uint64_t postponed_tx_count = 0;
849899
// pop pending state (reset to head block state)
@@ -865,7 +915,7 @@ namespace golos {
865915
}
866916

867917
try {
868-
auto temp_session = start_undo_session(true);
918+
auto temp_session = start_undo_session();
869919
_apply_transaction(tx);
870920
temp_session.squash();
871921

@@ -2931,7 +2981,7 @@ namespace golos {
29312981

29322982
void database::validate_transaction(const signed_transaction &trx) {
29332983
database::with_weak_write_lock([&]() {
2934-
auto session = start_undo_session(true);
2984+
auto session = start_undo_session();
29352985
_apply_transaction(trx);
29362986
session.undo();
29372987
});
@@ -3036,14 +3086,6 @@ namespace golos {
30363086
}
30373087
}
30383088

3039-
uint32_t free_gb = uint32_t(
3040-
get_free_memory() / (1024 * 1024 * 1024));
3041-
if ((free_gb < _last_free_gb_printed) ||
3042-
(free_gb > _last_free_gb_printed + 1)) {
3043-
ilog("Free memory is now ${n}G", ("n", free_gb));
3044-
_last_free_gb_printed = free_gb;
3045-
}
3046-
30473089
} FC_CAPTURE_AND_RETHROW((next_block))
30483090
}
30493091

‎libraries/chain/include/golos/chain/database.hpp

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ namespace golos {
9797
void reindex(const fc::path &data_dir, const fc::path &shared_mem_dir, uint64_t shared_file_size = (
9898
1024l * 1024l * 1024l * 8l));
9999

100+
void min_free_shared_memory_size(size_t);
101+
void inc_shared_memory_size(size_t);
102+
void block_num_check_free_size(uint32_t);
103+
void check_free_memory(bool skip_print, uint32_t current_block_num);
104+
100105
/**
101106
* @brief wipe Delete database from disk, and potentially the raw chain as well.
102107
* @param include_blocks If true, delete the raw chain as well as the database.
@@ -121,7 +126,7 @@ namespace golos {
121126

122127
uint32_t get_pow_summary_target() const;
123128

124-
block_id_type get_block_id_for_num( uint32_t block_num )const;
129+
block_id_type get_block_id_for_num( uint32_t block_num )const;
125130

126131
block_id_type find_block_id_for_num(uint32_t block_num) const;
127132

@@ -592,6 +597,11 @@ namespace golos {
592597

593598
uint32_t _last_free_gb_printed = 0;
594599

600+
size_t _inc_shared_memory_size = 0;
601+
size_t _min_free_shared_memory_size = 0;
602+
603+
uint32_t _block_num_check_free_memory = 1000;
604+
595605
flat_map<std::string, std::shared_ptr<custom_operation_interpreter>> _custom_operation_interpreters;
596606
std::string _json_schema;
597607
};

‎libraries/chain/include/golos/chain/generic_custom_operation_interpreter.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ namespace golos {
2828
}
2929

3030
void apply_operations(const vector<CustomOperationType> &custom_operations, const operation &outer_o) {
31-
auto plugin_session = this->_db.start_undo_session(true);
31+
auto plugin_session = this->_db.start_undo_session();
3232

3333
flat_set<account_name_type> outer_active;
3434
flat_set<account_name_type> outer_owner;

‎plugins/chain/plugin.cpp

Lines changed: 52 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ namespace chain {
3838
uint64_t write_wait_micro;
3939
uint32_t max_write_wait_retries;
4040

41+
size_t inc_shared_memory_size;
42+
size_t min_free_shared_memory_size;
43+
44+
uint32_t block_num_check_free_size = 0;
45+
4146
golos::chain::database db;
4247

4348
bool single_write_thread = false;
@@ -141,69 +146,54 @@ namespace chain {
141146
boost::program_options::options_description &cfg) {
142147
cfg.add_options()
143148
(
144-
"shared-file-dir",
145-
boost::program_options::value<boost::filesystem::path>()->default_value("blockchain"),
149+
"shared-file-dir", boost::program_options::value<boost::filesystem::path>()->default_value("blockchain"),
146150
"the location of the chain shared memory files (absolute path or relative to application data dir)"
147-
)
148-
(
149-
"shared-file-size",
150-
boost::program_options::value<std::string>()->default_value("64G"),
151-
"Size of the shared memory file. Default: 54G"
152-
)
153-
(
154-
"checkpoint,c",
155-
boost::program_options::value<std::vector<std::string>>()->composing(),
151+
) (
152+
"shared-file-size", boost::program_options::value<std::string>()->default_value("2G"),
153+
"Start size of the shared memory file. Default: 2G"
154+
) (
155+
"inc-shared-file-size", boost::program_options::value<std::string>()->default_value("2G"),
156+
"Increasing size on reaching limit of free space in shared memory file (see min-free-shared-file-size). Default: 2G"
157+
) (
158+
"min-free-shared-file-size", boost::program_options::value<std::string>()->default_value("500M"),
159+
"Minimum free space in shared memory file (see inc-shared-file-size). Default: 500M"
160+
) (
161+
"block-num-check-free-size", boost::program_options::value<uint32_t>()->default_value(1000),
162+
"Check free space in shared memory each N blocks. Default: 1000 (each 3000 seconds)."
163+
) (
164+
"checkpoint,c", boost::program_options::value<std::vector<std::string>>()->composing(),
156165
"Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints."
157-
)
158-
(
159-
"flush-state-interval",
160-
boost::program_options::value<uint32_t>(),
166+
) (
167+
"flush-state-interval", boost::program_options::value<uint32_t>(),
161168
"flush shared memory changes to disk every N blocks"
162-
)
163-
(
164-
"read-wait-micro",
165-
boost::program_options::value<uint64_t>(),
169+
) (
170+
"read-wait-micro", boost::program_options::value<uint64_t>(),
166171
"maximum microseconds for trying to get read lock"
167-
)
168-
(
169-
"max-read-wait-retries",
170-
boost::program_options::value<uint32_t>(),
172+
) (
173+
"max-read-wait-retries", boost::program_options::value<uint32_t>(),
171174
"maximum number of retries to get read lock"
172-
)
173-
(
174-
"write-wait-micro",
175-
boost::program_options::value<uint64_t>(),
175+
) (
176+
"write-wait-micro", boost::program_options::value<uint64_t>(),
176177
"maximum microseconds for trying to get write lock"
177-
)
178-
(
179-
"max-write-wait-retries",
180-
boost::program_options::value<uint32_t>(),
178+
) (
179+
"max-write-wait-retries", boost::program_options::value<uint32_t>(),
181180
"maximum number of retries to get write lock"
182-
)
183-
(
184-
"single-write-thread",
185-
boost::program_options::value<bool>()->default_value(false),
181+
) (
182+
"single-write-thread", boost::program_options::value<bool>()->default_value(false),
186183
"push blocks and transactions from one thread"
187184
);
188185
cli.add_options()
189186
(
190-
"replay-blockchain",
191-
boost::program_options::bool_switch()->default_value(false),
187+
"replay-blockchain", boost::program_options::bool_switch()->default_value(false),
192188
"clear chain database and replay all blocks"
193-
)
194-
(
195-
"resync-blockchain",
196-
boost::program_options::bool_switch()->default_value(false),
189+
) (
190+
"resync-blockchain", boost::program_options::bool_switch()->default_value(false),
197191
"clear chain database and block log"
198-
)
199-
(
200-
"check-locks",
201-
boost::program_options::bool_switch()->default_value(false),
192+
) (
193+
"check-locks", boost::program_options::bool_switch()->default_value(false),
202194
"Check correctness of chainbase locking"
203-
)
204-
(
205-
"validate-database-invariants",
206-
boost::program_options::bool_switch()->default_value(false),
195+
) (
196+
"validate-database-invariants", boost::program_options::bool_switch()->default_value(false),
207197
"Validate all supply invariants check out"
208198
);
209199
}
@@ -241,6 +231,12 @@ namespace chain {
241231
my->single_write_thread = options.at("single-write-thread").as<bool>();
242232

243233
my->shared_memory_size = fc::parse_size(options.at("shared-file-size").as<std::string>());
234+
my->inc_shared_memory_size = fc::parse_size(options.at("inc-shared-file-size").as<std::string>());
235+
my->min_free_shared_memory_size = fc::parse_size(options.at("min-free-shared-file-size").as<std::string>());
236+
237+
if (options.count("block-num-check-free-size")) {
238+
my->block_num_check_free_size = options.at("block-num-check-free-size").as<uint32_t>();
239+
}
244240

245241
my->replay = options.at("replay-blockchain").as<bool>();
246242
my->resync = options.at("resync-blockchain").as<bool>();
@@ -279,6 +275,13 @@ namespace chain {
279275
my->db.write_wait_micro(my->write_wait_micro);
280276
my->db.max_write_wait_retries(my->max_write_wait_retries);
281277

278+
my->db.inc_shared_memory_size(my->inc_shared_memory_size);
279+
my->db.min_free_shared_memory_size(my->min_free_shared_memory_size);
280+
281+
if (my->block_num_check_free_size) {
282+
my->db.block_num_check_free_size(my->block_num_check_free_size);
283+
}
284+
282285
if (my->replay) {
283286
ilog("Replaying blockchain on user request.");
284287
my->db.reindex(appbase::app().data_dir() / "blockchain", my->shared_memory_dir, my->shared_memory_size);

‎share/golosd/config/config.ini

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@ max-write-wait-retries = 3
4141

4242
single-write-thread = true
4343

44-
shared-file-size = 128G
44+
shared-file-size = 2G
45+
inc-shared-file-size = 2G
46+
min-free-shared-file-size = 500M
47+
block-num-check-free-size = 1000 # each 3000 seconds
4548

4649
plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network market_history account_by_key account_history chain_stats block_info raw_block
4750

‎share/golosd/config/config_debug.ini

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@ webserver-http-endpoint = 0.0.0.0:8090
3333
webserver-ws-endpoint = 0.0.0.0:8091
3434
rpc-endpoint = 0.0.0.0:2001
3535

36-
shared-file-size = 64G
36+
shared-file-size = 100M
37+
inc-shared-file-size = 100M
38+
min-free-shared-file-size = 50M
39+
block-num-check-free-size = 100 # each 300 seconds
3740

3841
read-wait-micro = 500000
3942
max-read-wait-retries = 2

‎share/golosd/config/config_stock_exchange.ini

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@ webserver-http-endpoint = 0.0.0.0:8090
3333
webserver-ws-endpoint = 0.0.0.0:8091
3434
rpc-endpoint = 0.0.0.0:2001
3535

36-
shared-file-size = 128G
36+
shared-file-size = 2G
37+
inc-shared-file-size = 2G
38+
min-free-shared-file-size = 500M
39+
block-num-check-free-size = 1000 # each 3000 seconds
3740

3841
read-wait-micro = 500000
3942
max-read-wait-retries = 3

0 commit comments

Comments
 (0)