From ac3eb1a7b86b4a5351210c4e2670e470f721b7df Mon Sep 17 00:00:00 2001 From: ton Date: Wed, 25 Sep 2019 17:50:58 +0400 Subject: [PATCH] tonlib updated - updated tonlib - added documentation - speed up full node synchronization --- CMakeLists.txt | 4 + crypto/fift/utils.cpp | 6 +- crypto/fift/utils.h | 2 +- crypto/smartcont/wallet.fif | 3 +- doc/smc-guidelines.txt | 77 ++++ example/android/build.sh | 2 +- tl/generate/scheme/tonlib_api.tl | 63 +++- tl/generate/scheme/tonlib_api.tlo | Bin 9548 -> 11624 bytes tonlib/CMakeLists.txt | 4 + tonlib/test/offline.cpp | 59 +++- tonlib/tonlib/GenericAccount.cpp | 2 +- tonlib/tonlib/KeyStorage.cpp | 37 +- tonlib/tonlib/KeyStorage.h | 5 +- tonlib/tonlib/LastBlockStorage.cpp | 18 + tonlib/tonlib/LastBlockStorage.h | 18 + tonlib/tonlib/Logging.cpp | 137 ++++++++ tonlib/tonlib/Logging.h | 49 +++ tonlib/tonlib/TestGiver.cpp | 4 +- tonlib/tonlib/TestWallet.cpp | 4 +- tonlib/tonlib/TonlibClient.cpp | 447 +++++++++++++++++++++--- tonlib/tonlib/TonlibClient.h | 32 +- tonlib/tonlib/Wallet.cpp | 86 +++++ tonlib/tonlib/Wallet.h | 38 ++ tonlib/tonlib/tonlib-cli.cpp | 62 +++- validator-session/validator-session.cpp | 2 +- validator/full-node-shard.cpp | 231 +++++++++++- validator/full-node-shard.hpp | 53 +++ validator/net/download-block-new.cpp | 4 +- validator/net/download-next-block.cpp | 56 +-- validator/net/download-next-block.hpp | 6 +- 30 files changed, 1351 insertions(+), 160 deletions(-) create mode 100644 doc/smc-guidelines.txt create mode 100644 tonlib/tonlib/Logging.cpp create mode 100644 tonlib/tonlib/Logging.h create mode 100644 tonlib/tonlib/Wallet.cpp create mode 100644 tonlib/tonlib/Wallet.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e07ca08..d71959b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -415,6 +415,10 @@ target_link_libraries(test-tonlib tdutils tdactor adnllite tl_api ton_crypto ton add_executable(test-tonlib-offline test/test-td-main.cpp ${TONLIB_OFFLINE_TEST_SOURCE}) target_link_libraries(test-tonlib-offline tdutils tdactor adnllite tl_api ton_crypto ton_block fift-lib tl_tonlib_api tonlib) + +if (NOT CMAKE_CROSSCOMPILING) + add_dependencies(test-tonlib-offline gen_fif) +endif() #END tonlib #BEGIN internal diff --git a/crypto/fift/utils.cpp b/crypto/fift/utils.cpp index e5f9a59..1c16b2c 100644 --- a/crypto/fift/utils.cpp +++ b/crypto/fift/utils.cpp @@ -163,13 +163,15 @@ td::Result create_mem_source_lookup(std::string main, std::s return create_source_lookup(main, need_preamble, need_asm, need_ton_util, fift_dir); } -td::Result> compile_asm(td::Slice asm_code, std::string fift_dir) { +td::Result> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) { std::stringstream ss; TRY_RESULT(source_lookup, - create_source_lookup(PSTRING() << "\"Asm.fif\" include\n<{ " << asm_code << "\n}>c boc>B \"res\" B>file", + create_source_lookup(PSTRING() << "\"Asm.fif\" include\n " << (is_raw ? "<{" : "") << asm_code << "\n" + << (is_raw ? "}>c" : "") << " boc>B \"res\" B>file", true, true, true, fift_dir)); TRY_RESULT(res, run_fift(std::move(source_lookup), &ss)); TRY_RESULT(boc, res.read_file("res")); return vm::std_boc_deserialize(std::move(boc.data)); } + } // namespace fift diff --git a/crypto/fift/utils.h b/crypto/fift/utils.h index e6ed145..e17eb8a 100644 --- a/crypto/fift/utils.h +++ b/crypto/fift/utils.h @@ -31,5 +31,5 @@ td::Result create_mem_source_lookup(std::string main, std::s bool need_ton_util = true); td::Result mem_run_fift(std::string source, std::vector args = {}, std::string fift_dir = ""); td::Result mem_run_fift(SourceLookup source_lookup, std::vector args); -td::Result> compile_asm(td::Slice asm_code, std::string fift_dir = ""); +td::Result> compile_asm(td::Slice asm_code, std::string fift_dir = "", bool is_raw = true); } // namespace fift diff --git a/crypto/smartcont/wallet.fif b/crypto/smartcont/wallet.fif index d9b8f00..1159796 100644 --- a/crypto/smartcont/wallet.fif +++ b/crypto/smartcont/wallet.fif @@ -16,6 +16,7 @@ $2 bounce parse-load-address =: bounce 2=: dest_addr $3 parse-int =: seqno $4 $>GR =: amount def? $5 { @' $5 } { "wallet-query" } cond constant savefile +3 constant send-mode // mode for SENDRAWMSG: +1 - sender pays fees, +2 - ignore errors file-base +".addr" load-address 2dup 2constant wallet_addr @@ -34,7 +35,7 @@ dest_addr 2dup bounce 7 + .Addr ." = " .addr - + dup ."signing message: " previous_transaction_id:in testWallet.initialAccountState public_key:string = testWallet.InitialAccountState; testWallet.accountState balance:int64 seqno:int32 last_transaction_id:internal.transactionId sync_utime:int53 = testWallet.AccountState; +wallet.initialAccountState public_key:string = wallet.InitialAccountState; +wallet.accountState balance:int64 seqno:int32 last_transaction_id:internal.transactionId sync_utime:int53 = wallet.AccountState; + testGiver.accountState balance:int64 seqno:int32 last_transaction_id:internal.transactionId sync_utime:int53= testGiver.AccountState; uninited.accountState balance:int64 last_transaction_id:internal.transactionId sync_utime:int53 = uninited.AccountState; -generic.initialAccountStateRaw initital_account_state:raw.initialAccountState = generic.InitialAccountState; -generic.initialAccountStateTestWallet initital_account_state:testWallet.initialAccountState = generic.InitialAccountState; +//generic.initialAccountStateRaw initital_account_state:raw.initialAccountState = generic.InitialAccountState; +//generic.initialAccountStateTestWallet initital_account_state:testWallet.initialAccountState = generic.InitialAccountState; +//generic.initialAccountStateWallet initital_account_state:wallet.initialAccountState = generic.InitialAccountState; generic.accountStateRaw account_state:raw.accountState = generic.AccountState; generic.accountStateTestWallet account_state:testWallet.accountState = generic.AccountState; +generic.accountStateWallet account_state:wallet.accountState = generic.AccountState; generic.accountStateTestGiver account_state:testGiver.accountState = generic.AccountState; generic.accountStateUninited account_state:uninited.accountState = generic.AccountState; +//generic.sendGramsResult sent_until:int53 = generic.SendGramsResult; + updateSendLiteServerQuery id:int64 data:bytes = Update; +//@class LogStream @description Describes a stream to which TDLib internal log is written + +//@description The log is written to stderr or an OS specific log +logStreamDefault = LogStream; + +//@description The log is written to a file @path Path to the file to where the internal TDLib log will be written @max_file_size Maximum size of the file to where the internal TDLib log is written before the file will be auto-rotated +logStreamFile path:string max_file_size:int53 = LogStream; + +//@description The log is written nowhere +logStreamEmpty = LogStream; + + +//@description Contains a TDLib internal log verbosity level @verbosity_level Log verbosity level +logVerbosityLevel verbosity_level:int32 = LogVerbosityLevel; + +//@description Contains a list of available TDLib internal log tags @tags List of log tags +logTags tags:vector = LogTags; + ---functions--- init options:options = Ok; @@ -61,7 +86,7 @@ close = Ok; options.setConfig config:string = Ok; createNewKey local_password:secureBytes mnemonic_password:secureBytes random_extra_seed:secureBytes = Key; -deleteKey public_key:string = Ok; +deleteKey key:key = Ok; exportKey input_key:inputKey = ExportedKey; exportPemKey input_key:inputKey key_password:secureBytes = ExportedPemKey; exportEncryptedKey input_key:inputKey key_password:secureBytes = ExportedEncryptedKey; @@ -83,6 +108,11 @@ testWallet.getAccountAddress initital_account_state:testWallet.initialAccountSta testWallet.getAccountState account_address:accountAddress = testWallet.AccountState; testWallet.sendGrams private_key:inputKey destination:accountAddress seqno:int32 amount:int64 message:bytes = Ok; +wallet.init private_key:inputKey = Ok; +wallet.getAccountAddress initital_account_state:wallet.initialAccountState = AccountAddress; +wallet.getAccountState account_address:accountAddress = wallet.AccountState; +wallet.sendGrams private_key:inputKey destination:accountAddress seqno:int32 valid_until:int53 amount:int64 message:bytes = Ok; + testGiver.getAccountState = testGiver.AccountState; testGiver.getAccountAddress = AccountAddress; testGiver.sendGrams destination:accountAddress seqno:int32 amount:int64 message:bytes = Ok; @@ -95,3 +125,30 @@ onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; runTests dir:string = Ok; + +//@description Sets new log stream for internal logging of TDLib. This is an offline method. Can be called before authorization. Can be called synchronously @log_stream New log stream +setLogStream log_stream:LogStream = Ok; + +//@description Returns information about currently used log stream for internal logging of TDLib. This is an offline method. Can be called before authorization. Can be called synchronously +getLogStream = LogStream; + +//@description Sets the verbosity level of the internal logging of TDLib. This is an offline method. Can be called before authorization. Can be called synchronously +//@new_verbosity_level New value of the verbosity level for logging. Value 0 corresponds to fatal errors, value 1 corresponds to errors, value 2 corresponds to warnings and debug warnings, value 3 corresponds to informational, value 4 corresponds to debug, value 5 corresponds to verbose debug, value greater than 5 and up to 1023 can be used to enable even more logging +setLogVerbosityLevel new_verbosity_level:int32 = Ok; + +//@description Returns current verbosity level of the internal logging of TDLib. This is an offline method. Can be called before authorization. Can be called synchronously +getLogVerbosityLevel = LogVerbosityLevel; + +//@description Returns list of available TDLib internal log tags, for example, ["actor", "binlog", "connections", "notifications", "proxy"]. This is an offline method. Can be called before authorization. Can be called synchronously +getLogTags = LogTags; + +//@description Sets the verbosity level for a specified TDLib internal log tag. This is an offline method. Can be called before authorization. Can be called synchronously +//@tag Logging tag to change verbosity level @new_verbosity_level New verbosity level; 1-1024 +setLogTagVerbosityLevel tag:string new_verbosity_level:int32 = Ok; + +//@description Returns current verbosity level for a specified TDLib internal log tag. This is an offline method. Can be called before authorization. Can be called synchronously @tag Logging tag to change verbosity level +getLogTagVerbosityLevel tag:string = LogVerbosityLevel; + +//@description Adds a message to TDLib internal log. This is an offline method. Can be called before authorization. Can be called synchronously +//@verbosity_level Minimum verbosity level needed for the message to be logged, 0-1023 @text Text of a message to log +addLogMessage verbosity_level:int32 text:string = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 40a4ccdce44ce004914d6b93a3f7b4c6155e02a7..0a810f234f4208ceb583c1e49916cce6390ac466 100644 GIT binary patch delta 1482 zcma)6Z%7ki7{6`jrgP`~bGoV3He8lrDkU)@(K2)BgiujPVO;L@HaMN^&P}68Q2J&$ zUXaQ!_PHqIz63=Pi1fiSLlgT@AQ53|MnYfu(EHxqt8*H7pPqZ}eSXjH`8~h)W*fd= zVOA!}&r;fR>?u>P&nj+mRj2vFe2WwTTn|MV;Jm?RS(4i(O(@&S@nXEsZdJ}sz0uye;7AD!y$uRmm<#_O!{~QZCTogkLJWUK>$)2?4;es zncojz6~tgPrA${UFQ-h<{EP9G=1hU~z6iRxenCR)FsJ32Y#$~XsWq0SR^57ya~g4K z@rb#z?ZoCCX1n%MNB}!^dUUYx+m{l$kCTEF1%I0JtxOL$;O|Dq{9@=Dpx_*Pm`}!> z{dIaJV;i%sr%xI@5En%)REO2VG3Kr#93&1_=eXdAk@ezli8CmwqPiqiDV$HluttgO zegW$a$4sn+2!jvHLGyxMT|y1xQG@GDg&IsTo=Uu*#LlCS$~*-;=mn8bR~gsU74Gkq zS|m;a?aIO^WuG;S+m5%0N5K#2sC--_@XidaVP}Tca7@d{(^jX+iIs@}3768b&pb-r2K8l)x%{3N3D$ zw?RaTI1!&At4ns^*ZaXDR;}6H71fZ&iS-FR<>!S*jaBhKPaa#u-4he($rUgoi~B|G z|ANrAuQTKvj*9V*wqcu!uxECHM3$(`zG=3zIlUm}-{)4~?bLiQt*)~w;+~-hBH{Kw zAo-1J!~mTR+PHPEc-kI=r-~YA#e`0y(|N&qyD(4tw;}b;CtM&(@iekUUb3g(HL@M) zF_`CWcHL;tF3;5JMbI0lk8nLAQiiZI!clkFk{;!RP{5C_W=MeVSr$%Z)9|nR@;@p` B22=n5 delta 362 zcmaD6b;gVL(QJJy1}IS8$m`Cu`2v#$BTL$<1J5S^VU}QJ0m=zXR1{X->^FC^bb4xD zYEfpgo@ZWWNoHb>V{&qSX{RM9n0{>LxI3e}_Ozyi{|d4Zq;JEQF8QmJZ25f#HtW}GRh vIjJS7-l>%g44M1hUYUGB++?$X%miqV%~64PYLlD?get_hash() == res->get_hash()); } +td::Ref get_wallet_source() { + return fift::compile_asm(load_source("smartcont/wallet-code.fif"), "", false).move_as_ok(); +} + +TEST(Tonlib, Wallet) { + LOG(ERROR) << td::base64_encode(std_boc_serialize(get_wallet_source()).move_as_ok()); + CHECK(get_wallet_source()->get_hash() == Wallet::get_init_code()->get_hash()); + // TODO: fix ater new-wallet supports new type of wallet + //auto fift_output = fift::mem_run_fift(load_source("smartcont/new-wallet.fif"), {"aba", "0"}).move_as_ok(); + + //auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data; + //auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet-query.boc").move_as_ok().data; + //auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet.addr").move_as_ok().data; + + //td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}}; + //auto pub_key = priv_key.get_public_key().move_as_ok(); + //auto init_state = TestWallet::get_init_state(pub_key); + //auto init_message = TestWallet::get_init_message(priv_key); + //auto address = GenericAccount::get_address(0, init_state); + + //CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); + + //td::Ref res = GenericAccount::create_ext_message(address, init_state, init_message); + + //LOG(ERROR) << "-------"; + //vm::load_cell_slice(res).print_rec(std::cerr); + //LOG(ERROR) << "-------"; + //vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr); + //CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash()); +} + TEST(Tonlib, TestGiver) { auto address = block::StdAddress::parse("-1:60c04141c6a7b96d68615e7a91d265ad0f3a9a922e9ae9c901d4fa83f5d3c0d0").move_as_ok(); @@ -346,9 +378,9 @@ TEST(Tonlib, KeysApi) { //importKey local_password:bytes mnemonic_password:bytes exported_key:exportedKey = Key; auto new_local_password = td::SecureString("new_local_password"); // import already existed key - sync_send(client, make_object(new_local_password.copy(), mnemonic_password.copy(), - make_object(copy_word_list()))) - .ensure_error(); + //sync_send(client, make_object(new_local_password.copy(), mnemonic_password.copy(), + //make_object(copy_word_list()))) + //.ensure_error(); { auto export_password = td::SecureString("export password"); @@ -361,7 +393,9 @@ TEST(Tonlib, KeysApi) { export_password.copy())) .move_as_ok(); - sync_send(client, make_object(key->public_key_)).move_as_ok(); + sync_send(client, + make_object(make_object(key->public_key_, key->secret_.copy()))) + .move_as_ok(); sync_send(client, make_object( new_local_password.copy(), wrong_export_password.copy(), @@ -374,10 +408,13 @@ TEST(Tonlib, KeysApi) { make_object(exported_encrypted_key->data_.copy()))) .move_as_ok(); CHECK(imported_encrypted_key->public_key_ == key->public_key_); + key = std::move(imported_encrypted_key); } //deleteKey public_key:bytes = Ok; - sync_send(client, make_object(key->public_key_)).move_as_ok(); + sync_send(client, + make_object(make_object(key->public_key_, key->secret_.copy()))) + .move_as_ok(); auto err1 = sync_send(client, make_object( new_local_password.copy(), td::SecureString("wrong password"), @@ -410,11 +447,13 @@ TEST(Tonlib, KeysApi) { LOG(ERROR) << to_string(exported_pem_key); //importPemKey exported_key:exportedPemKey key_password:bytes = Key; - sync_send(client, make_object( - new_local_password.copy(), pem_password.copy(), - make_object(exported_pem_key->pem_.copy()))) - .ensure_error(); - sync_send(client, make_object(key->public_key_)).move_as_ok(); + //sync_send(client, make_object( + //new_local_password.copy(), pem_password.copy(), + //make_object(exported_pem_key->pem_.copy()))) + //.ensure_error(); + sync_send(client, make_object( + make_object(imported_key->public_key_, imported_key->secret_.copy()))) + .move_as_ok(); sync_send(client, make_object( new_local_password.copy(), td::SecureString("wrong pem password"), make_object(exported_pem_key->pem_.copy()))) diff --git a/tonlib/tonlib/GenericAccount.cpp b/tonlib/tonlib/GenericAccount.cpp index 330393c..08ea33b 100644 --- a/tonlib/tonlib/GenericAccount.cpp +++ b/tonlib/tonlib/GenericAccount.cpp @@ -28,7 +28,7 @@ td::Ref GenericAccount::get_init_state(td::Ref code, td::Ref .finalize(); } block::StdAddress GenericAccount::get_address(ton::WorkchainId workchain_id, const td::Ref& init_state) { - return block::StdAddress(workchain_id, init_state->get_hash().bits(), false); + return block::StdAddress(workchain_id, init_state->get_hash().bits(), true /*bounce*/); } td::Ref GenericAccount::create_ext_message(const block::StdAddress& address, td::Ref new_state, td::Ref body) { diff --git a/tonlib/tonlib/KeyStorage.cpp b/tonlib/tonlib/KeyStorage.cpp index 1c25764..6137c0e 100644 --- a/tonlib/tonlib/KeyStorage.cpp +++ b/tonlib/tonlib/KeyStorage.cpp @@ -24,14 +24,23 @@ #include "td/utils/filesystem.h" #include "td/utils/port/path.h" +#include "td/utils/crypto.h" namespace tonlib { -std::string to_file_name(td::Slice public_key) { - return td::buffer_to_hex(public_key); +std::string to_file_name_old(const KeyStorage::Key &key) { + return td::buffer_to_hex(key.public_key); } -std::string KeyStorage::to_file_path(td::Slice public_key) { - return directory_ + TD_DIR_SLASH + to_file_name(public_key); +std::string KeyStorage::to_file_path_old(const Key &key) { + return directory_ + TD_DIR_SLASH + to_file_name_old(key); +} + +std::string to_file_name(const KeyStorage::Key &key) { + return td::buffer_to_hex(td::sha512(key.secret.as_slice()).substr(0, 32)); +} + +std::string KeyStorage::to_file_path(const Key &key) { + return directory_ + TD_DIR_SLASH + to_file_name(key); } td::Status KeyStorage::set_directory(std::string directory) { TRY_RESULT(path, td::realpath(directory)); @@ -52,8 +61,8 @@ td::Result KeyStorage::save_key(const DecryptedKey &decrypted_k auto size = encrypted_key.encrypted_data.size(); - LOG(ERROR) << "SAVE " << to_file_name(res.public_key); - TRY_RESULT(to_file, td::FileFd::open(to_file_path(res.public_key), td::FileFd::CreateNew | td::FileFd::Write)); + LOG(ERROR) << "SAVE " << to_file_name(res); + TRY_RESULT(to_file, td::FileFd::open(to_file_path(res), td::FileFd::CreateNew | td::FileFd::Write)); TRY_RESULT(written, to_file.write(encrypted_key.encrypted_data)); if (written != static_cast(size)) { return td::Status::Error(PSLICE() << "Failed to write file: written " << written << " bytes instead of " << size); @@ -74,7 +83,16 @@ td::Result KeyStorage::create_new_key(td::Slice local_password, } td::Result KeyStorage::export_decrypted_key(InputKey input_key) { - TRY_RESULT(encrypted_data, td::read_file_secure(to_file_path(input_key.key.public_key))); + auto r_encrypted_data = td::read_file_secure(to_file_path(input_key.key)); + if (r_encrypted_data.is_error()) { + r_encrypted_data = td::read_file_secure(to_file_path_old(input_key.key)); + if (r_encrypted_data.is_ok()) { + LOG(WARNING) << "Restore private from deprecated location " << to_file_path_old(input_key.key) << " --> " + << to_file_path(input_key.key); + td::rename(to_file_path_old(input_key.key), to_file_path(input_key.key)).ignore(); + } + } + TRY_RESULT(encrypted_data, std::move(r_encrypted_data)); EncryptedKey encrypted_key{std::move(encrypted_data), td::Ed25519::PublicKey(std::move(input_key.key.public_key)), std::move(input_key.key.secret)}; return encrypted_key.decrypt(std::move(input_key.local_password)); @@ -94,8 +112,8 @@ td::Result KeyStorage::load_private_key(InputKey input_k return std::move(private_key); } -td::Status KeyStorage::delete_key(td::Slice public_key) { - return td::unlink(to_file_path(public_key)); +td::Status KeyStorage::delete_key(const Key &key) { + return td::unlink(to_file_path(key)); } td::Result KeyStorage::import_key(td::Slice local_password, td::Slice mnemonic_password, @@ -122,6 +140,7 @@ td::Result KeyStorage::change_local_password(InputKey input_key Key res; res.public_key = std::move(input_key.key.public_key); res.secret = std::move(new_secret); + TRY_STATUS(td::copy_file(to_file_path(input_key.key), to_file_path(res))); return std::move(res); } diff --git a/tonlib/tonlib/KeyStorage.h b/tonlib/tonlib/KeyStorage.h index 305872b..f798280 100644 --- a/tonlib/tonlib/KeyStorage.h +++ b/tonlib/tonlib/KeyStorage.h @@ -57,7 +57,7 @@ class KeyStorage { td::Result export_encrypted_key(InputKey input_key, td::Slice key_password); td::Result change_local_password(InputKey input_key, td::Slice new_local_password); - td::Status delete_key(td::Slice public_key); + td::Status delete_key(const Key& key); td::Result import_key(td::Slice local_password, td::Slice mnemonic_password, ExportedKey exported_key); td::Result import_pem_key(td::Slice local_password, td::Slice key_password, ExportedPemKey exported_key); @@ -72,6 +72,7 @@ class KeyStorage { td::Result save_key(const DecryptedKey& mnemonic, td::Slice local_password); td::Result export_decrypted_key(InputKey input_key); - std::string to_file_path(td::Slice public_key); + std::string to_file_path(const Key& key); + std::string to_file_path_old(const Key& key); }; } // namespace tonlib diff --git a/tonlib/tonlib/LastBlockStorage.cpp b/tonlib/tonlib/LastBlockStorage.cpp index fa979a3..684b2c0 100644 --- a/tonlib/tonlib/LastBlockStorage.cpp +++ b/tonlib/tonlib/LastBlockStorage.cpp @@ -1,3 +1,21 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ #include "LastBlockStorage.h" #include "td/utils/as.h" diff --git a/tonlib/tonlib/LastBlockStorage.h b/tonlib/tonlib/LastBlockStorage.h index fa920ef..d8cbd3f 100644 --- a/tonlib/tonlib/LastBlockStorage.h +++ b/tonlib/tonlib/LastBlockStorage.h @@ -1,3 +1,21 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ #pragma once #include "tonlib/LastBlock.h" diff --git a/tonlib/tonlib/Logging.cpp b/tonlib/tonlib/Logging.cpp new file mode 100644 index 0000000..043fe01 --- /dev/null +++ b/tonlib/tonlib/Logging.cpp @@ -0,0 +1,137 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "Logging.h" + +#include "auto/tl/tonlib_api.h" + +#include "td/utils/FileLog.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/misc.h" + +#include +#include +#include + +namespace tonlib { + +static std::mutex logging_mutex; +static td::FileLog file_log; +static td::TsLog ts_log(&file_log); +static td::NullLog null_log; + +td::int32 VERBOSITY_NAME(abc) = VERBOSITY_NAME(DEBUG); +td::int32 VERBOSITY_NAME(bcd) = VERBOSITY_NAME(DEBUG); +#define ADD_TAG(tag) \ + { #tag, &VERBOSITY_NAME(tag) } +static const std::map log_tags{ADD_TAG(abc), ADD_TAG(bcd)}; +#undef ADD_TAG + +td::Status Logging::set_current_stream(tonlib_api::object_ptr stream) { + if (stream == nullptr) { + return td::Status::Error("Log stream must not be empty"); + } + + std::lock_guard lock(logging_mutex); + switch (stream->get_id()) { + case tonlib_api::logStreamDefault::ID: + td::log_interface = td::default_log_interface; + return td::Status::OK(); + case tonlib_api::logStreamFile::ID: { + auto file_stream = tonlib_api::move_object_as(stream); + auto max_log_file_size = file_stream->max_file_size_; + if (max_log_file_size <= 0) { + return td::Status::Error("Max log file size should be positive"); + } + + TRY_STATUS(file_log.init(file_stream->path_, max_log_file_size)); + std::atomic_thread_fence(std::memory_order_release); // better than nothing + td::log_interface = &ts_log; + return td::Status::OK(); + } + case tonlib_api::logStreamEmpty::ID: + td::log_interface = &null_log; + return td::Status::OK(); + default: + UNREACHABLE(); + return td::Status::OK(); + } +} + +td::Result> Logging::get_current_stream() { + std::lock_guard lock(logging_mutex); + if (td::log_interface == td::default_log_interface) { + return tonlib_api::make_object(); + } + if (td::log_interface == &null_log) { + return tonlib_api::make_object(); + } + if (td::log_interface == &ts_log) { + return tonlib_api::make_object(file_log.get_path().str(), + file_log.get_rotate_threshold()); + } + return td::Status::Error("Log stream is unrecognized"); +} + +td::Status Logging::set_verbosity_level(int new_verbosity_level) { + std::lock_guard lock(logging_mutex); + if (0 <= new_verbosity_level && new_verbosity_level <= VERBOSITY_NAME(NEVER)) { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(FATAL) + new_verbosity_level); + return td::Status::OK(); + } + + return td::Status::Error("Wrong new verbosity level specified"); +} + +int Logging::get_verbosity_level() { + std::lock_guard lock(logging_mutex); + return GET_VERBOSITY_LEVEL(); +} + +td::vector Logging::get_tags() { + return transform(log_tags, [](auto &tag) { return tag.first.str(); }); +} + +td::Status Logging::set_tag_verbosity_level(td::Slice tag, int new_verbosity_level) { + auto it = log_tags.find(tag); + if (it == log_tags.end()) { + return td::Status::Error("Log tag is not found"); + } + + std::lock_guard lock(logging_mutex); + *it->second = td::clamp(new_verbosity_level, 1, VERBOSITY_NAME(NEVER)); + return td::Status::OK(); +} + +td::Result Logging::get_tag_verbosity_level(td::Slice tag) { + auto it = log_tags.find(tag); + if (it == log_tags.end()) { + return td::Status::Error("Log tag is not found"); + } + + std::lock_guard lock(logging_mutex); + return *it->second; +} + +void Logging::add_message(int log_verbosity_level, td::Slice message) { + int VERBOSITY_NAME(client) = td::clamp(log_verbosity_level, 0, VERBOSITY_NAME(NEVER)); + VLOG(client) << message; +} + +} // namespace tonlib diff --git a/tonlib/tonlib/Logging.h b/tonlib/tonlib/Logging.h new file mode 100644 index 0000000..9dbda9d --- /dev/null +++ b/tonlib/tonlib/Logging.h @@ -0,0 +1,49 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "auto/tl/tonlib_api.h" + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace tonlib { +namespace tonlib_api = ton::tonlib_api; + +class Logging { + public: + static td::Status set_current_stream(tonlib_api::object_ptr stream); + + static td::Result> get_current_stream(); + + static td::Status set_verbosity_level(int new_verbosity_level); + + static int get_verbosity_level(); + + static std::vector get_tags(); + + static td::Status set_tag_verbosity_level(td::Slice tag, int new_verbosity_level); + + static td::Result get_tag_verbosity_level(td::Slice tag); + + static void add_message(int log_verbosity_level, td::Slice message); +}; + +} // namespace tonlib diff --git a/tonlib/tonlib/TestGiver.cpp b/tonlib/tonlib/TestGiver.cpp index 9ea58e5..50c6857 100644 --- a/tonlib/tonlib/TestGiver.cpp +++ b/tonlib/tonlib/TestGiver.cpp @@ -34,7 +34,7 @@ vm::CellHash TestGiver::get_init_code_hash() { td::Ref TestGiver::make_a_gift_message(td::uint32 seqno, td::uint64 gramms, td::Slice message, const block::StdAddress& dest_address) { - CHECK(message.size() <= 128); + CHECK(message.size() <= 124); td::BigInt256 dest_addr; dest_addr.import_bits(dest_address.addr.as_bitslice()); vm::CellBuilder cb; @@ -45,7 +45,7 @@ td::Ref TestGiver::make_a_gift_message(td::uint32 seqno, td::uint64 gr .store_int256(dest_addr, 256); block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes(message).finalize(); + auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("\0\0\0\0", 4).store_bytes(message).finalize(); return vm::CellBuilder().store_long(seqno, 32).store_long(1, 8).store_ref(message_inner).finalize(); } } // namespace tonlib diff --git a/tonlib/tonlib/TestWallet.cpp b/tonlib/tonlib/TestWallet.cpp index c092bac..0bbf920 100644 --- a/tonlib/tonlib/TestWallet.cpp +++ b/tonlib/tonlib/TestWallet.cpp @@ -40,7 +40,7 @@ td::Ref TestWallet::get_init_message(const td::Ed25519::PrivateKey& pr td::Ref TestWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, td::int64 gramms, td::Slice message, const block::StdAddress& dest_address) { - CHECK(message.size() <= 128); + CHECK(message.size() <= 124); td::BigInt256 dest_addr; dest_addr.import_bits(dest_address.addr.as_bitslice()); vm::CellBuilder cb; @@ -48,7 +48,7 @@ td::Ref TestWallet::make_a_gift_message(const td::Ed25519::PrivateKey& .store_long(dest_address.workchain, 8) .store_int256(dest_addr, 256); block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes(message).finalize(); + auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("\0\0\0\0", 4).store_bytes(message).finalize(); auto message_outer = vm::CellBuilder().store_long(seqno, 32).store_long(1, 8).store_ref(message_inner).finalize(); std::string seq_no(4, 0); auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index ea0c6c8..7c03817 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -22,7 +22,9 @@ #include "tonlib/ExtClientOutbound.h" #include "tonlib/GenericAccount.h" #include "tonlib/LastBlock.h" +#include "tonlib/Logging.h" #include "tonlib/TestWallet.h" +#include "tonlib/Wallet.h" #include "tonlib/TestGiver.h" #include "tonlib/utils.h" #include "tonlib/keys/Mnemonic.h" @@ -412,8 +414,17 @@ bool TonlibClient::is_static_request(td::int32 id) { case tonlib_api::runTests::ID: case tonlib_api::raw_getAccountAddress::ID: case tonlib_api::testWallet_getAccountAddress::ID: + case tonlib_api::wallet_getAccountAddress::ID: case tonlib_api::testGiver_getAccountAddress::ID: case tonlib_api::getBip39Hints::ID: + case tonlib_api::setLogStream::ID: + case tonlib_api::getLogStream::ID: + case tonlib_api::setLogVerbosityLevel::ID: + case tonlib_api::getLogVerbosityLevel::ID: + case tonlib_api::getLogTags::ID: + case tonlib_api::setLogTagVerbosityLevel::ID: + case tonlib_api::getLogTagVerbosityLevel::ID: + case tonlib_api::addLogMessage::ID: return true; default: return false; @@ -449,6 +460,12 @@ td::Result get_account_address(const tonlib_api::testWallet_i return GenericAccount::get_address(0 /*zerochain*/, TestWallet::get_init_state(key)); } +td::Result get_account_address(const tonlib_api::wallet_initialAccountState& test_wallet_state) { + TRY_RESULT(key_bytes, block::PublicKey::parse(test_wallet_state.public_key_)); + auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); + return GenericAccount::get_address(0 /*zerochain*/, Wallet::get_init_state(key)); +} + tonlib_api::object_ptr TonlibClient::do_static_request( const tonlib_api::raw_getAccountAddress& request) { auto r_account_address = get_account_address(*request.initital_account_state_); @@ -465,6 +482,14 @@ tonlib_api::object_ptr TonlibClient::do_static_request( } return tonlib_api::make_object(r_account_address.ok().rserialize()); } +tonlib_api::object_ptr TonlibClient::do_static_request( + const tonlib_api::wallet_getAccountAddress& request) { + auto r_account_address = get_account_address(*request.initital_account_state_); + if (r_account_address.is_error()) { + return status_to_tonlib_api(r_account_address.error()); + } + return tonlib_api::make_object(r_account_address.ok().rserialize()); +} tonlib_api::object_ptr TonlibClient::do_static_request( const tonlib_api::testGiver_getAccountAddress& request) { return tonlib_api::make_object(TestGiver::address().rserialize()); @@ -601,6 +626,10 @@ td::Result> to_raw_message_or_th if (body->size() % 8 == 0) { body_message = std::string(body->size() / 8, 0); body->prefetch_bytes(td::MutableSlice(body_message).ubegin(), body->size() / 8); + if (body_message.size() >= 4 && body_message[0] == 0 && body_message[1] == 0 && body_message[2] == 0 && + body_message[3] == 0) { + body_message = body_message.substr(4); + } } return tonlib_api::make_object(std::move(src), std::move(dest), balance, @@ -714,6 +743,21 @@ td::Result> to_testW raw_state.balance, static_cast(seqno), to_transaction_id(raw_state.info), raw_state.sync_utime); } +td::Result> to_wallet_accountState( + RawAccountState&& raw_state) { + if (raw_state.data.is_null()) { + return td::Status::Error(400, "Not a Wallet"); + } + auto ref = raw_state.data->prefetch_ref(); + auto cs = vm::load_cell_slice(std::move(ref)); + auto seqno = cs.fetch_ulong(32); + if (seqno == cs.fetch_ulong_eof) { + return td::Status::Error("Failed to parse seq_no"); + } + return tonlib_api::make_object( + raw_state.balance, static_cast(seqno), to_transaction_id(raw_state.info), raw_state.sync_utime); +} + td::Result> to_testGiver_accountState( RawAccountState&& raw_state) { if (raw_state.data.is_null()) { @@ -742,6 +786,10 @@ td::Result> to_generic_ TRY_RESULT(test_wallet, to_testWallet_accountState(std::move(raw_state))); return tonlib_api::make_object(std::move(test_wallet)); } + if (code_hash == Wallet::get_init_code_hash()) { + TRY_RESULT(wallet, to_wallet_accountState(std::move(raw_state))); + return tonlib_api::make_object(std::move(wallet)); + } if (code_hash == TestGiver::get_init_code_hash()) { TRY_RESULT(test_wallet, to_testGiver_accountState(std::move(raw_state))); return tonlib_api::make_object(std::move(test_wallet)); @@ -858,7 +906,7 @@ td::Status TonlibClient::do_request(const tonlib_api::testWallet_sendGrams& requ if (!request.private_key_) { return td::Status::Error(400, "Field private_key must not be empty"); } - if (request.message_.size() > 128) { + if (request.message_.size() > 124) { return td::Status::Error(400, "Message is too long"); } TRY_RESULT(account_address, block::StdAddress::parse(request.destination_->account_address_)); @@ -896,13 +944,79 @@ td::Status TonlibClient::do_request(tonlib_api::testWallet_getAccountState& requ return td::Status::OK(); } +// Wallet +td::Status TonlibClient::do_request(const tonlib_api::wallet_init& request, + td::Promise>&& promise) { + if (!request.private_key_) { + return td::Status::Error(400, "Field private_key must not be empty"); + } + TRY_RESULT(input_key, from_tonlib(*request.private_key_)); + auto init_state = Wallet::get_init_state(td::Ed25519::PublicKey(input_key.key.public_key.copy())); + auto address = GenericAccount::get_address(0 /*zerochain*/, init_state); + TRY_RESULT(private_key, key_storage_.load_private_key(std::move(input_key))); + auto init_message = Wallet::get_init_message(td::Ed25519::PrivateKey(std::move(private_key.private_key))); + return do_request( + tonlib_api::raw_sendMessage(tonlib_api::make_object(address.rserialize()), + vm::std_boc_serialize(init_state).move_as_ok().as_slice().str(), + vm::std_boc_serialize(init_message).move_as_ok().as_slice().str()), + std::move(promise)); +} + +td::Status TonlibClient::do_request(const tonlib_api::wallet_sendGrams& request, + td::Promise>&& promise) { + if (!request.destination_) { + return td::Status::Error(400, "Field destination must not be empty"); + } + if (!request.private_key_) { + return td::Status::Error(400, "Field private_key must not be empty"); + } + if (request.message_.size() > 124) { + return td::Status::Error(400, "Message is too long"); + } + TRY_RESULT(valid_until, td::narrow_cast_safe(request.valid_until_)); + TRY_RESULT(account_address, block::StdAddress::parse(request.destination_->account_address_)); + account_address.bounceable = false; + TRY_RESULT(input_key, from_tonlib(*request.private_key_)); + auto address = GenericAccount::get_address( + 0 /*zerochain*/, Wallet::get_init_state(td::Ed25519::PublicKey(input_key.key.public_key.copy()))); + TRY_RESULT(private_key, key_storage_.load_private_key(std::move(input_key))); + return do_request( + tonlib_api::raw_sendMessage( + tonlib_api::make_object(address.rserialize()), "", + vm::std_boc_serialize(Wallet::make_a_gift_message(td::Ed25519::PrivateKey(std::move(private_key.private_key)), + request.seqno_, valid_until, request.amount_, + request.message_, account_address)) + .move_as_ok() + .as_slice() + .str()), + std::move(promise)); +} + +td::Status TonlibClient::do_request(tonlib_api::wallet_getAccountState& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return td::Status::Error(400, "Field account_address must not be empty"); + } + TRY_RESULT(account_address, block::StdAddress::parse(request.account_address_->account_address_)); + td::actor::create_actor( + "GetAccountState", client_.get_client(), std::move(account_address), + [promise = std::move(promise)](td::Result r_state) mutable { + if (r_state.is_error()) { + return promise.set_error(r_state.move_as_error()); + } + promise.set_result(to_wallet_accountState(r_state.move_as_ok())); + }) + .release(); + return td::Status::OK(); +} + // TestGiver td::Status TonlibClient::do_request(const tonlib_api::testGiver_sendGrams& request, td::Promise>&& promise) { if (!request.destination_) { return td::Status::Error(400, "Field destination must not be empty"); } - if (request.message_.size() > 128) { + if (request.message_.size() > 124) { return td::Status::Error(400, "Message is too long"); } TRY_RESULT(account_address, block::StdAddress::parse(request.destination_->account_address_)); @@ -949,61 +1063,219 @@ td::Status TonlibClient::do_request(const tonlib_api::generic_getAccountState& r return td::Status::OK(); } +class TonlibQueryActor : public td::actor::Actor { + public: + TonlibQueryActor(td::actor::ActorShared client) : client_(std::move(client)) { + } + template + void send_query(QueryT query, td::Promise promise) { + td::actor::send_lambda(client_, + [self = client_.get(), query = std::move(query), promise = std::move(promise)]() mutable { + auto status = self.get_actor_unsafe().do_request(query, std::move(promise)); + if (status.is_error()) { + promise.set_error(std::move(status)); + } + }); + } + + private: + td::actor::ActorShared client_; +}; + +class GenericSendGrams : public TonlibQueryActor { + public: + GenericSendGrams(td::actor::ActorShared client, tonlib_api::generic_sendGrams send_grams, + td::Promise>&& promise) + : TonlibQueryActor(std::move(client)), send_grams_(std::move(send_grams)), promise_(std::move(promise)) { + timeout_ = td::Timestamp::in(15); + } + + private: + tonlib_api::generic_sendGrams send_grams_; + td::Promise> promise_; + + enum class SourceAction { Wait, Init, WaitInited, Ok } source_action_ = SourceAction::Wait; + tonlib_api::object_ptr source_state_; + block::StdAddress source_address_; + td::Timestamp source_next_get_state_; + td::Timestamp timeout_; + bool has_source_state_query_{false}; + + tonlib_api::object_ptr destination_state_; + bool is_destination_bounce_{false}; + + void check(td::Status status) { + if (status.is_error()) { + LOG(ERROR) << status; + promise_.set_error(std::move(status)); + return stop(); + } + } + + void start_up() override { + check(do_start_up()); + } + + td::Status do_start_up() { + if (!send_grams_.destination_) { + return td::Status::Error(400, "Field destination must not be empty"); + } + TRY_RESULT(destination_address, block::StdAddress::parse(send_grams_.destination_->account_address_)); + is_destination_bounce_ = destination_address.bounceable; + + if (!send_grams_.source_) { + return td::Status::Error(400, "Field source must not be empty"); + } + TRY_RESULT(source_address, block::StdAddress::parse(send_grams_.source_->account_address_)); + source_address_ = std::move(source_address); + + send_query(tonlib_api::generic_getAccountState( + tonlib_api::make_object(send_grams_.source_->account_address_)), + [actor_id = actor_id(this)](auto r_res) { + send_closure(actor_id, &GenericSendGrams::on_source_state, std::move(r_res)); + }); + send_query(tonlib_api::generic_getAccountState( + tonlib_api::make_object(send_grams_.destination_->account_address_)), + [actor_id = actor_id(this)](auto r_res) { + send_closure(actor_id, &GenericSendGrams::on_destination_state, std::move(r_res)); + }); + return do_loop(); + } + + static tonlib_api::object_ptr clone(const tonlib_api::object_ptr& ptr) { + if (!ptr) { + return nullptr; + } + return tonlib_api::make_object(ptr->public_key_, ptr->secret_.copy()); + } + + static tonlib_api::object_ptr clone(const tonlib_api::object_ptr& ptr) { + if (!ptr) { + return nullptr; + } + return tonlib_api::make_object(clone(ptr->key_), ptr->local_password_.copy()); + } + + void on_source_state(td::Result> r_state) { + check(do_on_source_state(std::move(r_state))); + } + + td::Status do_on_source_state(td::Result> r_state) { + has_source_state_query_ = false; + TRY_RESULT(state, std::move(r_state)); + source_state_ = std::move(state); + if (source_action_ == SourceAction::Wait) { + source_action_ = SourceAction::Ok; + if (false && source_state_->get_id() == tonlib_api::generic_accountStateUninited::ID && + send_grams_.private_key_ && send_grams_.private_key_->key_) { + TRY_RESULT(key_bytes, block::PublicKey::parse(send_grams_.private_key_->key_->public_key_)); + auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); + auto addr = GenericAccount::get_address(0 /*zerochain*/, TestWallet::get_init_state(key)); + if (addr.addr == source_address_.addr) { + source_action_ = SourceAction::Init; + send_query(tonlib_api::testWallet_init(clone(send_grams_.private_key_)), + [actor_id = actor_id(this)](auto r_res) { + send_closure(actor_id, &GenericSendGrams::on_source_init, std::move(r_res)); + }); + } + } + } else if (source_action_ == SourceAction::WaitInited) { + if (source_state_->get_id() != tonlib_api::generic_accountStateUninited::ID) { + source_action_ = SourceAction::Ok; + } + } + return do_loop(); + } + + void on_source_init(td::Result> r_ok) { + do_on_source_init(std::move(r_ok)); + } + + td::Status do_on_source_init(td::Result> r_ok) { + TRY_RESULT(ok, std::move(r_ok)); + source_action_ = SourceAction::WaitInited; + return do_loop(); + } + + void on_destination_state(td::Result> r_state) { + check(do_on_destination_state(std::move(r_state))); + } + + td::Status do_on_destination_state(td::Result> r_state) { + TRY_RESULT(state, std::move(r_state)); + destination_state_ = std::move(state); + if (destination_state_->get_id() == tonlib_api::generic_accountStateUninited::ID) { + //return td::Status::Error("Transfer to uninited wallet"); + } + return do_loop(); + } + + void alarm() override { + check(do_loop()); + } + td::Status do_loop() { + if (timeout_.is_in_past()) { + return td::Status::Error("Timeout"); + } + alarm_timestamp().relax(timeout_); + if (source_action_ == SourceAction::WaitInited && !has_source_state_query_) { + if (source_next_get_state_.is_in_past()) { + source_next_get_state_ = td::Timestamp::in(1); + has_source_state_query_ = true; + send_query(tonlib_api::generic_getAccountState( + tonlib_api::make_object(send_grams_.source_->account_address_)), + [actor_id = actor_id(this)](auto r_res) { + send_closure(actor_id, &GenericSendGrams::on_source_state, std::move(r_res)); + }); + } else { + alarm_timestamp().relax(source_next_get_state_); + } + } + if (source_action_ != SourceAction::Ok || !destination_state_) { + return td::Status::OK(); + } + downcast_call(*source_state_, + td::overloaded( + [&](tonlib_api::generic_accountStateTestGiver& test_giver_state) { + send_query(tonlib_api::testGiver_sendGrams( + std::move(send_grams_.destination_), test_giver_state.account_state_->seqno_, + send_grams_.amount_, std::move(send_grams_.message_)), + std::move(promise_)); + stop(); + }, + [&](tonlib_api::generic_accountStateTestWallet& test_wallet_state) { + send_query(tonlib_api::testWallet_sendGrams( + std::move(send_grams_.private_key_), std::move(send_grams_.destination_), + test_wallet_state.account_state_->seqno_, send_grams_.amount_, + std::move(send_grams_.message_)), + std::move(promise_)); + stop(); + }, + [&](tonlib_api::generic_accountStateWallet& test_wallet_state) { + send_query(tonlib_api::wallet_sendGrams( + std::move(send_grams_.private_key_), std::move(send_grams_.destination_), + test_wallet_state.account_state_->seqno_, std::numeric_limits::max(), + send_grams_.amount_, std::move(send_grams_.message_)), + std::move(promise_)); + stop(); + }, + [&](tonlib_api::generic_accountStateUninited&) { + promise_.set_error(td::Status::Error(400, "Account is not inited")); + stop(); + }, + [&](tonlib_api::generic_accountStateRaw&) { + promise_.set_error(td::Status::Error(400, "Unknown account type")); + stop(); + })); + return td::Status::OK(); + } +}; + td::Status TonlibClient::do_request(tonlib_api::generic_sendGrams& request, td::Promise>&& promise) { - TRY_RESULT(account_address, block::StdAddress::parse(request.source_->account_address_)); - LOG(INFO) << "Send " << request.amount_ << " nanograms from " << account_address.rserialize(); - td::actor::create_actor( - "GetAccountState", client_.get_client(), std::move(account_address), - [promise = std::move(promise), self = this, actor_id = td::actor::actor_id(), - private_key = std::move(request.private_key_), destination = std::move(request.destination_), - amount = request.amount_, message = std::move(request.message_)](td::Result r_state) mutable { - if (r_state.is_error()) { - return promise.set_error(r_state.move_as_error()); - } - auto rr_state = to_generic_accountState(r_state.move_as_ok()); - if (rr_state.is_error()) { - return promise.set_error(rr_state.move_as_error()); - } - auto state = rr_state.move_as_ok(); - - downcast_call(*state, td::overloaded( - [&](tonlib_api::generic_accountStateTestGiver& test_giver_state) { - send_lambda(actor_id, - [promise = std::move(promise), self, - query = tonlib_api::testGiver_sendGrams( - std::move(destination), test_giver_state.account_state_->seqno_, - amount, std::move(message))]() mutable { - LOG(INFO) << "Send " << to_string(query); - auto status = self->do_request(query, std::move(promise)); - if (status.is_error()) { - CHECK(promise); - promise.set_error(std::move(status)); - } - }); - return; - }, - [&](tonlib_api::generic_accountStateTestWallet& test_wallet_state) { - send_lambda(actor_id, [promise = std::move(promise), self, - query = tonlib_api::testWallet_sendGrams( - std::move(private_key), std::move(destination), - test_wallet_state.account_state_->seqno_, amount, - std::move(message))]() mutable { - auto status = self->do_request(query, std::move(promise)); - if (status.is_error()) { - CHECK(promise); - promise.set_error(std::move(status)); - } - }); - }, - [&](tonlib_api::generic_accountStateUninited&) { - promise.set_error(td::Status::Error(400, "Account is not inited")); - }, - [&](tonlib_api::generic_accountStateRaw&) { - promise.set_error(td::Status::Error(400, "Unknown account type")); - })); - }) - .release(); + auto id = actor_id_++; + actors_[id] = td::actor::create_actor("GenericSendGrams", actor_shared(this, id), + std::move(request), std::move(promise)); return td::Status::OK(); } @@ -1029,8 +1301,14 @@ td::Status TonlibClient::do_request(const tonlib_api::exportKey& request, td::Status TonlibClient::do_request(const tonlib_api::deleteKey& request, td::Promise>&& promise) { - TRY_RESULT(key_bytes, block::PublicKey::parse(request.public_key_)); - TRY_STATUS(key_storage_.delete_key(key_bytes.key)); + if (!request.key_) { + return td::Status::Error(400, "Field key must not be empty"); + } + TRY_RESULT(key_bytes, block::PublicKey::parse(request.key_->public_key_)); + KeyStorage::Key key; + key.public_key = td::SecureString(key_bytes.key); + key.secret = std::move(request.key_->secret_); + TRY_STATUS(key_storage_.delete_key(key)); promise.set_value(tonlib_api::make_object()); return td::Status::OK(); } @@ -1141,4 +1419,59 @@ td::Status TonlibClient::do_request(const tonlib_api::onLiteServerQueryError& re return td::Status::OK(); } +tonlib_api::object_ptr TonlibClient::do_static_request(tonlib_api::setLogStream& request) { + auto result = Logging::set_current_stream(std::move(request.log_stream_)); + if (result.is_ok()) { + return tonlib_api::make_object(); + } else { + return tonlib_api::make_object(400, result.message().str()); + } +} +tonlib_api::object_ptr TonlibClient::do_static_request(const tonlib_api::getLogStream& request) { + auto result = Logging::get_current_stream(); + if (result.is_ok()) { + return result.move_as_ok(); + } else { + return tonlib_api::make_object(400, result.error().message().str()); + } +} +tonlib_api::object_ptr TonlibClient::do_static_request( + const tonlib_api::setLogVerbosityLevel& request) { + auto result = Logging::set_verbosity_level(static_cast(request.new_verbosity_level_)); + if (result.is_ok()) { + return tonlib_api::make_object(); + } else { + return tonlib_api::make_object(400, result.message().str()); + } +} +tonlib_api::object_ptr TonlibClient::do_static_request( + const tonlib_api::setLogTagVerbosityLevel& request) { + auto result = Logging::set_tag_verbosity_level(request.tag_, static_cast(request.new_verbosity_level_)); + if (result.is_ok()) { + return tonlib_api::make_object(); + } else { + return tonlib_api::make_object(400, result.message().str()); + } +} +tonlib_api::object_ptr TonlibClient::do_static_request( + const tonlib_api::getLogVerbosityLevel& request) { + return tonlib_api::make_object(Logging::get_verbosity_level()); +} +tonlib_api::object_ptr TonlibClient::do_static_request( + const tonlib_api::getLogTagVerbosityLevel& request) { + auto result = Logging::get_tag_verbosity_level(request.tag_); + if (result.is_ok()) { + return tonlib_api::make_object(result.ok()); + } else { + return tonlib_api::make_object(400, result.error().message().str()); + } +} +tonlib_api::object_ptr TonlibClient::do_static_request(const tonlib_api::getLogTags& request) { + return tonlib_api::make_object(Logging::get_tags()); +} +tonlib_api::object_ptr TonlibClient::do_static_request(const tonlib_api::addLogMessage& request) { + Logging::add_message(request.verbosity_level_, request.text_); + return tonlib_api::make_object(); +} + } // namespace tonlib diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index a45fd6f..5ac81c2 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -28,6 +28,8 @@ #include "td/actor/actor.h" +#include + namespace tonlib { class TonlibClient : public td::actor::Actor { public: @@ -58,6 +60,9 @@ class TonlibClient : public td::actor::Actor { td::actor::ActorOwn raw_last_block_; ExtClient client_; + std::map> actors_; + td::int64 actor_id_{1}; + ExtClientRef get_client_ref(); void init_ext_client(); void init_last_block(); @@ -65,12 +70,17 @@ class TonlibClient : public td::actor::Actor { bool is_closing_{false}; td::uint32 ref_cnt_{1}; void hangup_shared() override { - ref_cnt_--; + auto it = actors_.find(get_link_token()); + if (it != actors_.end()) { + actors_.erase(it); + } else { + ref_cnt_--; + } try_stop(); } void hangup() override; void try_stop() { - if (is_closing_ && ref_cnt_ == 0) { + if (is_closing_ && ref_cnt_ == 0 && actors_.empty()) { stop(); } } @@ -86,8 +96,19 @@ class TonlibClient : public td::actor::Actor { static object_ptr do_static_request(const tonlib_api::runTests& request); static object_ptr do_static_request(const tonlib_api::raw_getAccountAddress& request); static object_ptr do_static_request(const tonlib_api::testWallet_getAccountAddress& request); + static object_ptr do_static_request(const tonlib_api::wallet_getAccountAddress& request); static object_ptr do_static_request(const tonlib_api::testGiver_getAccountAddress& request); static object_ptr do_static_request(tonlib_api::getBip39Hints& request); + + static object_ptr do_static_request(tonlib_api::setLogStream& request); + static object_ptr do_static_request(const tonlib_api::getLogStream& request); + static object_ptr do_static_request(const tonlib_api::setLogVerbosityLevel& request); + static object_ptr do_static_request(const tonlib_api::setLogTagVerbosityLevel& request); + static object_ptr do_static_request(const tonlib_api::getLogVerbosityLevel& request); + static object_ptr do_static_request(const tonlib_api::getLogTagVerbosityLevel& request); + static object_ptr do_static_request(const tonlib_api::getLogTags& request); + static object_ptr do_static_request(const tonlib_api::addLogMessage& request); + template td::Status do_request(const T& request, P&& promise) { return td::Status::Error(400, "Function is unsupported"); @@ -112,6 +133,11 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(tonlib_api::testWallet_getAccountState& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::wallet_init& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::wallet_sendGrams& request, td::Promise>&& promise); + td::Status do_request(tonlib_api::wallet_getAccountState& request, + td::Promise>&& promise); + td::Status do_request(const tonlib_api::testGiver_getAccountState& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::testGiver_sendGrams& request, @@ -145,5 +171,7 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); void proxy_request(td::int64 query_id, std::string data); + + friend class TonlibQueryActor; }; } // namespace tonlib diff --git a/tonlib/tonlib/Wallet.cpp b/tonlib/tonlib/Wallet.cpp new file mode 100644 index 0000000..c0758ca --- /dev/null +++ b/tonlib/tonlib/Wallet.cpp @@ -0,0 +1,86 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "tonlib/Wallet.h" +#include "tonlib/GenericAccount.h" +#include "tonlib/utils.h" + +#include "vm/boc.h" +#include "td/utils/base64.h" + +#include + +namespace tonlib { +td::Ref Wallet::get_init_state(const td::Ed25519::PublicKey& public_key) { + auto code = get_init_code(); + auto data = get_init_data(public_key); + return GenericAccount::get_init_state(std::move(code), std::move(data)); +} + +td::Ref Wallet::get_init_message(const td::Ed25519::PrivateKey& private_key) { + td::uint32 seqno = 0; + td::uint32 valid_until = std::numeric_limits::max(); + auto signature = + private_key + .sign(vm::CellBuilder().store_long(seqno, 32).store_long(valid_until, 32).finalize()->get_hash().as_slice()) + .move_as_ok(); + return vm::CellBuilder().store_bytes(signature).store_long(seqno, 32).store_long(valid_until, 32).finalize(); +} + +td::Ref Wallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, + td::uint32 valid_until, td::int64 gramms, td::Slice message, + const block::StdAddress& dest_address) { + CHECK(message.size() <= 124); + td::BigInt256 dest_addr; + dest_addr.import_bits(dest_address.addr.as_bitslice()); + vm::CellBuilder cb; + cb.append_cellslice(binary_bitstring_to_cellslice("b{010000100}").move_as_ok()) + .store_long(dest_address.workchain, 8) + .store_int256(dest_addr, 256); + block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); + auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("\0\0\0\0", 4).store_bytes(message).finalize(); + auto message_outer = vm::CellBuilder() + .store_long(seqno, 32) + .store_long(valid_until, 32) + .store_long(1, 8) + .store_ref(message_inner) + .finalize(); + std::string seq_no(4, 0); + auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); + return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); +} + +td::Ref Wallet::get_init_code() { + static auto res = [] { + auto serialized_code = td::base64_decode( + "te6ccgEEBgEAAAAAaAABFP8A9KQT9KDyyAsBAgEgAgMCAUgEBQCA8oMI1xgg0x/TH/gjErnyY+1E0NMf0//" + "RUTG68qED+QFUEEL5EPKi+AACkyDXSpbTB9QC+wDo0aTIyx/L/8ntVAAE0DAAEaCZL9qJoa4WPw==") + .move_as_ok(); + return vm::std_boc_deserialize(serialized_code).move_as_ok(); + }(); + return res; +} + +vm::CellHash Wallet::get_init_code_hash() { + return get_init_code()->get_hash(); +} + +td::Ref Wallet::get_init_data(const td::Ed25519::PublicKey& public_key) { + return vm::CellBuilder().store_long(0, 32).store_bytes(public_key.as_octet_string()).finalize(); +} +} // namespace tonlib diff --git a/tonlib/tonlib/Wallet.h b/tonlib/tonlib/Wallet.h new file mode 100644 index 0000000..2f2d8a8 --- /dev/null +++ b/tonlib/tonlib/Wallet.h @@ -0,0 +1,38 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "vm/cells.h" +#include "Ed25519.h" +#include "block/block.h" + +namespace tonlib { +class Wallet { + public: + static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key); + static td::Ref get_init_message(const td::Ed25519::PrivateKey& private_key); + static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, + td::uint32 valid_until, td::int64 gramms, td::Slice message, + const block::StdAddress& dest_address); + + static td::Ref get_init_code(); + static vm::CellHash get_init_code_hash(); + static td::Ref get_init_data(const td::Ed25519::PublicKey& public_key); +}; +} // namespace tonlib diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index f6ce888..191adba 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -27,6 +27,7 @@ class TonlibCli : public td::actor::Actor { std::string config; std::string key_dir{"."}; bool use_callbacks_for_network{false}; + bool use_simple_wallet{false}; }; TonlibCli(Options options) : options_(std::move(options)) { } @@ -281,7 +282,7 @@ class TonlibCli : public td::actor::Actor { info.public_key = key->public_key_; info.secret = std::move(key->secret_); keys_.push_back(std::move(info)); - export_key(info.public_key, keys_.size() - 1, std::move(password)); + //export_key(key->public_key_, keys_.size() - 1, std::move(password)); store_keys(); }); } @@ -388,8 +389,11 @@ class TonlibCli : public td::actor::Actor { auto r_key_i = to_key_i(key); using tonlib_api::make_object; if (r_key_i.is_ok()) { - auto obj = tonlib::TonlibClient::static_request(make_object( - make_object(keys_[r_key_i.ok()].public_key))); + auto obj = options_.use_simple_wallet + ? tonlib::TonlibClient::static_request(make_object( + make_object(keys_[r_key_i.ok()].public_key))) + : tonlib::TonlibClient::static_request(make_object( + make_object(keys_[r_key_i.ok()].public_key))); if (obj->get_id() != tonlib_api::error::ID) { Address res; res.address = ton::move_tl_object_as(obj); @@ -476,12 +480,19 @@ class TonlibCli : public td::actor::Actor { using tonlib_api::make_object; send_query(make_object(td::SecureString(password), td::SecureString(), make_object(std::move(words))), - [](auto r_res) { + [this, password = td::SecureString(password)](auto r_res) { if (r_res.is_error()) { td::TerminalIO::out() << "Can't import key " << r_res.error() << "\n"; return; } - td::TerminalIO::out() << to_string(r_res.ok()); + auto key = r_res.move_as_ok(); + LOG(ERROR) << to_string(key); + KeyInfo info; + info.public_key = key->public_key_; + info.secret = std::move(key->secret_); + keys_.push_back(std::move(info)); + export_key(key->public_key_, keys_.size() - 1, std::move(password)); + store_keys(); }); } @@ -618,7 +629,7 @@ class TonlibCli : public td::actor::Actor { std::move(to.address), grams, message.str()), [](auto r_res) { if (r_res.is_error()) { - td::TerminalIO::out() << "Can't get state: " << r_res.error() << "\n"; + td::TerminalIO::out() << "Can't transfer: " << r_res.error() << "\n"; return; } td::TerminalIO::out() << to_string(r_res.ok()); @@ -648,16 +659,29 @@ class TonlibCli : public td::actor::Actor { void init_simple_wallet(std::string key, size_t key_i, td::Slice password) { using tonlib_api::make_object; - send_query(make_object(make_object( - make_object(keys_[key_i].public_key, keys_[key_i].secret.copy()), - td::SecureString(password))), - [key = std::move(key)](auto r_res) { - if (r_res.is_error()) { - td::TerminalIO::out() << "Can't init wallet with key: [" << key << "] " << r_res.error() << "\n"; - return; - } - td::TerminalIO::out() << to_string(r_res.ok()); - }); + if (options_.use_simple_wallet) { + send_query(make_object(make_object( + make_object(keys_[key_i].public_key, keys_[key_i].secret.copy()), + td::SecureString(password))), + [key = std::move(key)](auto r_res) { + if (r_res.is_error()) { + td::TerminalIO::out() << "Can't init wallet with key: [" << key << "] " << r_res.error() << "\n"; + return; + } + td::TerminalIO::out() << to_string(r_res.ok()); + }); + } else { + send_query(make_object(make_object( + make_object(keys_[key_i].public_key, keys_[key_i].secret.copy()), + td::SecureString(password))), + [key = std::move(key)](auto r_res) { + if (r_res.is_error()) { + td::TerminalIO::out() << "Can't init wallet with key: [" << key << "] " << r_res.error() << "\n"; + return; + } + td::TerminalIO::out() << to_string(r_res.ok()); + }); + } } void get_hints(td::Slice prefix) { @@ -705,10 +729,14 @@ int main(int argc, char* argv[]) { options.config = std::move(data); return td::Status::OK(); }); - p.add_option('c', "use_callbacks_for_network (for debug)", "do not use this", [&]() { + p.add_option('c', "use-callbacks-for-network", "do not use this", [&]() { options.use_callbacks_for_network = true; return td::Status::OK(); }); + p.add_option('S', "use-simple-wallet", "do not use this", [&]() { + options.use_simple_wallet = true; + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 4b7e8d5..5d5f707 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -296,7 +296,7 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data if (round_id < real_state_->cur_round_seqno()) { block = real_state_->get_committed_block(description(), round_id); if (!block || SentBlock::get_block_id(block) != id) { - promise.set_error(td::Status::Error(ErrorCode::notready, "wrong block")); + promise.set_error(td::Status::Error(ErrorCode::notready, "wrong block in old round")); return; } } else { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 2aeb1a2..130d344 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -32,6 +32,8 @@ #include "net/download-proof.hpp" #include "net/get-next-key-blocks.hpp" +#include "td/utils/Random.h" + #include "common/delay.h" namespace ton { @@ -40,6 +42,29 @@ namespace validator { namespace fullnode { +Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; + +void Neighbour::update_proto_version(const ton_api::tonNode_capabilities &q) { + proto_version = q.version_; + capabilities = q.capabilities_; +} + +void Neighbour::query_success(td::Clocks::Duration t) { + unreliability--; + if (unreliability < 0) { + unreliability = 0; + } + update_roundtrip(t); +} + +void Neighbour::query_failed() { + unreliability++; +} + +void Neighbour::update_roundtrip(td::Clocks::Duration t) { + roundtrip = (t + roundtrip) * 0.5; +} + void FullNodeShardImpl::create_overlay() { class Callback : public overlay::Overlays::Callback { public: @@ -80,15 +105,19 @@ void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise("downloadnext", adnl_id_, overlay_id_, handle_->id(), - ton::adnl::AdnlNodeIdShort::zero(), download_next_priority(), timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, std::move(promise)) + + auto &b = choose_neighbour(); + if (!b.adnl_id.is_zero() && b.proto_version >= 1) { + VLOG(FULL_NODE_DEBUG) << "using new download method with adnlid=" << b.adnl_id; + td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_->id(), b.adnl_id, + download_next_priority(), timeout, validator_manager_, rldp_, overlays_, + adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } else { - td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_, download_next_priority(), - timeout, validator_manager_, rldp_, overlays_, adnl_, client_, - std::move(promise)) + VLOG(FULL_NODE_DEBUG) << "using old download method with adnlid=" << b.adnl_id; + td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_, b.adnl_id, + download_next_priority(), timeout, validator_manager_, rldp_, overlays_, + adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } } @@ -549,15 +578,18 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - if (use_new_download()) { - td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), - priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, - std::move(promise)) + auto &b = choose_neighbour(); + if (!b.adnl_id.is_zero() && b.proto_version >= 1) { + VLOG(FULL_NODE_DEBUG) << "new block download"; + td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, + validator_manager_, rldp_, overlays_, adnl_, client_, + create_neighbour_promise(b, std::move(promise))) .release(); } else { - td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), - priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, - std::move(promise)) + VLOG(FULL_NODE_DEBUG) << "old block download"; + td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, + validator_manager_, rldp_, overlays_, adnl_, client_, + create_neighbour_promise(b, std::move(promise))) .release(); } } @@ -580,25 +612,28 @@ void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt mast void FullNodeShardImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor("downloadproofreq", block_id, false, adnl_id_, overlay_id_, - adnl::AdnlNodeIdShort::zero(), priority, timeout, validator_manager_, rldp_, - overlays_, adnl_, client_, std::move(promise)) + auto &b = choose_neighbour(); + td::actor::create_actor("downloadproofreq", block_id, false, adnl_id_, overlay_id_, b.adnl_id, + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + create_neighbour_promise(b, std::move(promise))) .release(); } void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { + auto &b = choose_neighbour(); td::actor::create_actor("downloadproofreq", block_id, true, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), priority, timeout, validator_manager_, rldp_, - overlays_, adnl_, client_, std::move(promise)) + overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) { + auto &b = choose_neighbour(); td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), 1, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, - std::move(promise)) + create_neighbour_promise(b, std::move(promise))) .release(); } @@ -619,6 +654,14 @@ void FullNodeShardImpl::alarm() { } sync_completed_at_ = td::Timestamp::never(); } + if (reload_neighbours_at_ && reload_neighbours_at_.is_in_past()) { + reload_neighbours(); + reload_neighbours_at_ = td::Timestamp::in(td::Random::fast(10.0, 30.0)); + } + if (ping_neighbours_at_ && ping_neighbours_at_.is_in_past()) { + ping_neighbours(); + ping_neighbours_at_ = td::Timestamp::in(td::Random::fast(0.5, 1.0)); + } if (update_certificate_at_ && update_certificate_at_.is_in_past()) { if (!sign_cert_by_.is_zero()) { sign_new_certificate(sign_cert_by_); @@ -629,6 +672,8 @@ void FullNodeShardImpl::alarm() { } alarm_timestamp().relax(sync_completed_at_); alarm_timestamp().relax(update_certificate_at_); + alarm_timestamp().relax(reload_neighbours_at_); + alarm_timestamp().relax(ping_neighbours_at_); } void FullNodeShardImpl::start_up() { @@ -642,6 +687,11 @@ void FullNodeShardImpl::start_up() { rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_fec_broadcast_size()}; create_overlay(); + + reload_neighbours_at_ = td::Timestamp::now(); + ping_neighbours_at_ = td::Timestamp::now(); + alarm_timestamp().relax(reload_neighbours_at_); + alarm_timestamp().relax(ping_neighbours_at_); } } @@ -701,6 +751,149 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ } } +void FullNodeShardImpl::reload_neighbours() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + return; + } + auto vec = R.move_as_ok(); + if (vec.size() == 0) { + return; + } else { + td::actor::send_closure(SelfId, &FullNodeShardImpl::got_neighbours, std::move(vec)); + } + }); + td::actor::send_closure(overlays_, &overlay::Overlays::get_overlay_random_peers, adnl_id_, overlay_id_, + max_neighbours(), std::move(P)); +} + +void FullNodeShardImpl::got_neighbours(std::vector vec) { + bool ex = false; + + for (auto &el : vec) { + auto it = neighbours_.find(el); + if (it != neighbours_.end()) { + continue; + } + if (neighbours_.size() == max_neighbours()) { + adnl::AdnlNodeIdShort a = adnl::AdnlNodeIdShort::zero(); + adnl::AdnlNodeIdShort b = adnl::AdnlNodeIdShort::zero(); + td::uint32 cnt = 0; + double u = 0; + for (auto &n : neighbours_) { + if (n.second.unreliability > u) { + u = n.second.unreliability; + a = n.first; + } + if (td::Random::fast(0, cnt++) == 0) { + b = n.first; + } + } + + if (u > stop_unreliability()) { + neighbours_.erase(a); + } else { + neighbours_.erase(b); + ex = true; + } + } + neighbours_.emplace(el, Neighbour{el}); + if (ex) { + break; + } + } +} + +const Neighbour &FullNodeShardImpl::choose_neighbour() const { + if (neighbours_.size() == 0) { + return Neighbour::zero; + } + + const Neighbour *best = nullptr; + td::uint32 sum = 0; + + for (auto &x : neighbours_) { + td::uint32 unr = static_cast(x.second.unreliability); + + if (x.second.proto_version < proto_version()) { + unr += 4; + } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { + unr += 2; + } + + auto f = static_cast(fail_unreliability()); + + if (unr <= f) { + auto w = 1 << (f - unr); + sum += w; + if (td::Random::fast(0, sum - 1) <= w - 1) { + best = &x.second; + } + } + } + return best ? *best : Neighbour::zero; +} + +void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, td::Clocks::Duration t, bool success) { + auto it = neighbours_.find(adnl_id); + if (it != neighbours_.end()) { + if (success) { + it->second.query_success(t); + } else { + it->second.query_failed(); + } + } +} + +void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, td::Clocks::Duration t, + td::BufferSlice data) { + auto it = neighbours_.find(adnl_id); + if (it == neighbours_.end()) { + return; + } + auto F = fetch_tl_object(std::move(data), true); + if (F.is_error()) { + it->second.query_failed(); + } else { + it->second.update_proto_version(*F.move_as_ok().get()); + it->second.update_roundtrip(t); + } +} + +void FullNodeShardImpl::ping_neighbours() { + if (neighbours_.size() == 0) { + return; + } + td::uint32 max_cnt = 6; + if (max_cnt > neighbours_.size()) { + max_cnt = td::narrow_cast(neighbours_.size()); + } + auto it = neighbours_.lower_bound(last_pinged_neighbour_); + while (max_cnt > 0) { + if (it == neighbours_.end()) { + it = neighbours_.begin(); + } + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), start_time = td::Time::now(), id = it->first](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, + td::Time::now() - start_time, false); + } else { + td::actor::send_closure(SelfId, &FullNodeShardImpl::got_neighbour_capabilities, id, + td::Time::now() - start_time, R.move_as_ok()); + } + }); + auto q = create_serialize_tl_object(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_query, it->first, adnl_id_, overlay_id_, + "get_prepare_block", std::move(P), td::Timestamp::in(1.0), std::move(q)); + + last_pinged_neighbour_ = it->first; + it++; + max_cnt--; + } +} + FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 283cf60..1724b9a 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -26,6 +26,25 @@ namespace validator { namespace fullnode { +struct Neighbour { + adnl::AdnlNodeIdShort adnl_id; + td::uint32 proto_version = 0; + td::uint64 capabilities = 0; + td::Clocks::Duration roundtrip = 0; + td::Clocks::Duration roundtrip_relax_at = 0; + double roundtrip_weight = 0; + double unreliability = 0; + + Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { + } + void update_proto_version(const ton_api::tonNode_capabilities &q); + void query_success(td::Clocks::Duration t); + void query_failed(); + void update_roundtrip(td::Clocks::Duration t); + + static Neighbour zero; +}; + class FullNodeShardImpl : public FullNodeShard { public: WorkchainId get_workchain() const override { @@ -47,6 +66,15 @@ class FullNodeShardImpl : public FullNodeShard { static constexpr td::uint64 proto_capabilities() { return 0; } + static constexpr td::uint32 max_neighbours() { + return 16; + } + static constexpr double stop_unreliability() { + return 5.0; + } + static constexpr double fail_unreliability() { + return 10.0; + } void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; @@ -130,6 +158,26 @@ class FullNodeShardImpl : public FullNodeShard { void sign_new_certificate(PublicKeyHash sign_by); void signed_new_certificate(ton::overlay::Certificate cert); + void ping_neighbours(); + void reload_neighbours(); + void got_neighbours(std::vector res); + void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, td::Clocks::Duration t, bool success); + void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, td::Clocks::Duration t, td::BufferSlice data); + const Neighbour &choose_neighbour() const; + + template + td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p) { + return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), + ts = td::Time::now()](td::Result R) mutable { + if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { + td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); + } else { + td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, true); + } + p.set_result(std::move(R)); + }); + } + FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, @@ -167,6 +215,11 @@ class FullNodeShardImpl : public FullNodeShard { std::shared_ptr cert_; overlay::OverlayPrivacyRules rules_; + + std::map neighbours_; + td::Timestamp reload_neighbours_at_; + td::Timestamp ping_neighbours_at_; + adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); }; } // namespace fullnode diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index 6ce7c60..145a081 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -226,7 +226,7 @@ void DownloadBlockNew::got_data(td::BufferSlice data) { [&](ton_api::tonNode_dataFullEmpty &x) { abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block")); }, - [&](ton_api::tonNode_dataFull &x) { + [&, self = this](ton_api::tonNode_dataFull &x) { if (!allow_partial_proof_ && x.is_link_) { abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block")); return; @@ -243,7 +243,7 @@ void DownloadBlockNew::got_data(td::BufferSlice data) { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(self)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("received bad proof: ")); diff --git a/validator/net/download-next-block.cpp b/validator/net/download-next-block.cpp index 2512e82..95e37ac 100644 --- a/validator/net/download-next-block.cpp +++ b/validator/net/download-next-block.cpp @@ -29,7 +29,8 @@ namespace validator { namespace fullnode { DownloadNextBlock::DownloadNextBlock(adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, - BlockHandle prev, td::uint32 priority, td::Timestamp timeout, + BlockHandle prev, adnl::AdnlNodeIdShort download_from, td::uint32 priority, + td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, @@ -38,6 +39,7 @@ DownloadNextBlock::DownloadNextBlock(adnl::AdnlNodeIdShort local_id, overlay::Ov : local_id_(local_id) , overlay_id_(overlay_id) , prev_(prev) + , download_from_(download_from) , priority_(priority) , timeout_(timeout) , validator_manager_(validator_manager) @@ -51,11 +53,11 @@ DownloadNextBlock::DownloadNextBlock(adnl::AdnlNodeIdShort local_id, overlay::Ov void DownloadNextBlock::abort_query(td::Status reason) { if (promise_) { if (reason.code() == ErrorCode::notready || reason.code() == ErrorCode::timeout) { - VLOG(FULL_NODE_DEBUG) << "failed to download next block after " << prev_->id() << " from " << node_ << ": " - << reason; + VLOG(FULL_NODE_DEBUG) << "failed to download next block after " << prev_->id() << " from " << download_from_ + << ": " << reason; } else { - VLOG(FULL_NODE_INFO) << "failed to download next block after " << prev_->id() << " from " << node_ << ": " - << reason; + VLOG(FULL_NODE_INFO) << "failed to download next block after " << prev_->id() << " from " << download_from_ + << ": " << reason; } promise_.set_error(std::move(reason)); } @@ -79,30 +81,34 @@ void DownloadNextBlock::start_up() { return; } if (!client_.empty()) { - got_node(node_); + got_node(download_from_); return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadNextBlock::abort_query, R.move_as_error()); - } else { - auto vec = R.move_as_ok(); - if (vec.size() == 0) { - td::actor::send_closure(SelfId, &DownloadNextBlock::abort_query, - td::Status::Error(ErrorCode::notready, "no neighbours found")); + if (download_from_.is_zero()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadNextBlock::abort_query, R.move_as_error()); } else { - td::actor::send_closure(SelfId, &DownloadNextBlock::got_node, vec[0]); + auto vec = R.move_as_ok(); + if (vec.size() == 0) { + td::actor::send_closure(SelfId, &DownloadNextBlock::abort_query, + td::Status::Error(ErrorCode::notready, "no neighbours found")); + } else { + td::actor::send_closure(SelfId, &DownloadNextBlock::got_node, vec[0]); + } } - } - }); + }); - td::actor::send_closure(overlays_, &overlay::Overlays::get_overlay_random_peers, local_id_, overlay_id_, 1, - std::move(P)); + td::actor::send_closure(overlays_, &overlay::Overlays::get_overlay_random_peers, local_id_, overlay_id_, 1, + std::move(P)); + } else { + got_node(download_from_); + } } void DownloadNextBlock::got_node(adnl::AdnlNodeIdShort id) { - node_ = id; + download_from_ = id; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -114,8 +120,8 @@ void DownloadNextBlock::got_node(adnl::AdnlNodeIdShort id) { auto query = create_serialize_tl_object(create_tl_block_id(prev_->id())); if (client_.empty()) { - td::actor::send_closure(overlays_, &overlay::Overlays::send_query, id, local_id_, overlay_id_, "get_prepare", - std::move(P), td::Timestamp::in(1.0), std::move(query)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_query, download_from_, local_id_, overlay_id_, + "get_prepare", std::move(P), td::Timestamp::in(1.0), std::move(query)); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_prepare", create_serialize_tl_object_suffix(std::move(query)), @@ -149,9 +155,9 @@ void DownloadNextBlock::got_next_node_handle(BlockHandle handle) { void DownloadNextBlock::finish_query() { if (promise_) { - td::actor::create_actor("downloadnext", next_block_id_, local_id_, overlay_id_, prev_, node_, - priority_, timeout_, validator_manager_, rldp_, overlays_, adnl_, client_, - std::move(promise_)) + td::actor::create_actor("downloadnext", next_block_id_, local_id_, overlay_id_, prev_, + download_from_, priority_, timeout_, validator_manager_, rldp_, overlays_, + adnl_, client_, std::move(promise_)) .release(); } stop(); diff --git a/validator/net/download-next-block.hpp b/validator/net/download-next-block.hpp index 8dc768b..c7646d4 100644 --- a/validator/net/download-next-block.hpp +++ b/validator/net/download-next-block.hpp @@ -34,7 +34,7 @@ namespace fullnode { class DownloadNextBlock : public td::actor::Actor { public: DownloadNextBlock(adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, BlockHandle prev, - td::uint32 priority, td::Timestamp timeout, + adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, @@ -57,6 +57,8 @@ class DownloadNextBlock : public td::actor::Actor { BlockIdExt next_block_id_; + adnl::AdnlNodeIdShort download_from_ = adnl::AdnlNodeIdShort::zero(); + td::uint32 priority_; td::Timestamp timeout_; @@ -66,8 +68,6 @@ class DownloadNextBlock : public td::actor::Actor { td::actor::ActorId adnl_; td::actor::ActorId client_; td::Promise promise_; - - adnl::AdnlNodeIdShort node_ = adnl::AdnlNodeIdShort::zero(); }; } // namespace fullnode