diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake index 932659d030f0..0daaf8c5a877 100644 --- a/cmake/ssl.cmake +++ b/cmake/ssl.cmake @@ -451,10 +451,10 @@ FUNCTION(FIND_CUSTOM_OPENSSL) FIND_LIBRARY(OPENSSL_LIBRARY NAMES ssl libssl ssleay32 ssleay32MD - HINTS ${OPENSSL_ROOT_DIR}/lib ${OPENSSL_ROOT_DIR}/lib64) + HINTS ${OPENSSL_ROOT_DIR}/lib64 ${OPENSSL_ROOT_DIR}/lib) FIND_LIBRARY(CRYPTO_LIBRARY NAMES crypto libcrypto libeay32 - HINTS ${OPENSSL_ROOT_DIR}/lib ${OPENSSL_ROOT_DIR}/lib64) + HINTS ${OPENSSL_ROOT_DIR}/lib64 ${OPENSSL_ROOT_DIR}/lib) IF(OPENSSL_INCLUDE_DIR) FIND_OPENSSL_VERSION() diff --git a/mysql-test/suite/ndb/r/ndbinfo_plans.result b/mysql-test/suite/ndb/r/ndbinfo_plans.result index 80e7d9231abf..afdc0ebab0fc 100644 --- a/mysql-test/suite/ndb/r/ndbinfo_plans.result +++ b/mysql-test/suite/ndb/r/ndbinfo_plans.result @@ -75,7 +75,7 @@ ndb$disk_write_speed_base 488 48 ndb$diskpagebuffer 10 64 ndb$diskstat 10 48 ndb$diskstats_1sec 200 52 -ndb$error_messages 865 52 +ndb$error_messages 867 52 ndb$frag_locks 400 96 ndb$frag_mem_use 400 100 ndb$frag_operations 400 192 diff --git a/mysql-test/suite/ndb_binlog/r/ndb_binlog_basic.result b/mysql-test/suite/ndb_binlog/r/ndb_binlog_basic.result index 17d2839c7bee..824f55a1f5d9 100644 --- a/mysql-test/suite/ndb_binlog/r/ndb_binlog_basic.result +++ b/mysql-test/suite/ndb_binlog/r/ndb_binlog_basic.result @@ -48,7 +48,7 @@ inserts updates deletes flush logs; purge binary logs before now(); Warnings: -Warning 1868 file ./binlog.000002 was not purged because it is the active log file. +Warning 1869 file ./binlog.000002 was not purged because it is the active log file. select count(*) from mysql.ndb_binlog_index; count(*) 0 diff --git a/mysql-test/suite/ndb_quota/ndb_basic.result b/mysql-test/suite/ndb_quota/ndb_basic.result index 4f41376cc5b6..69e7e3e28057 100644 --- a/mysql-test/suite/ndb_quota/ndb_basic.result +++ b/mysql-test/suite/ndb_quota/ndb_basic.result @@ -1,222 +1,4 @@ DATABASE QUOTA SET successfully executed -SHOW GLOBAL STATUS LIKE 'ndb\_%'; -Variable_name Value -Ndb_api_adaptive_send_deferred_count # -Ndb_api_adaptive_send_deferred_count_replica # -Ndb_api_adaptive_send_deferred_count_session # -Ndb_api_adaptive_send_deferred_count_slave # -Ndb_api_adaptive_send_forced_count # -Ndb_api_adaptive_send_forced_count_replica # -Ndb_api_adaptive_send_forced_count_session # -Ndb_api_adaptive_send_forced_count_slave # -Ndb_api_adaptive_send_unforced_count # -Ndb_api_adaptive_send_unforced_count_replica # -Ndb_api_adaptive_send_unforced_count_session # -Ndb_api_adaptive_send_unforced_count_slave # -Ndb_api_bytes_received_count # -Ndb_api_bytes_received_count_replica # -Ndb_api_bytes_received_count_session # -Ndb_api_bytes_received_count_slave # -Ndb_api_bytes_sent_count # -Ndb_api_bytes_sent_count_replica # -Ndb_api_bytes_sent_count_session # -Ndb_api_bytes_sent_count_slave # -Ndb_api_event_bytes_count # -Ndb_api_event_bytes_count_injector # -Ndb_api_event_data_count # -Ndb_api_event_data_count_injector # -Ndb_api_event_nondata_count # -Ndb_api_event_nondata_count_injector # -Ndb_api_pk_op_count # -Ndb_api_pk_op_count_replica # -Ndb_api_pk_op_count_session # -Ndb_api_pk_op_count_slave # -Ndb_api_pruned_scan_count # -Ndb_api_pruned_scan_count_replica # -Ndb_api_pruned_scan_count_session # -Ndb_api_pruned_scan_count_slave # -Ndb_api_range_scan_count # -Ndb_api_range_scan_count_replica # -Ndb_api_range_scan_count_session # -Ndb_api_range_scan_count_slave # -Ndb_api_read_row_count # -Ndb_api_read_row_count_replica # -Ndb_api_read_row_count_session # -Ndb_api_read_row_count_slave # -Ndb_api_scan_batch_count # -Ndb_api_scan_batch_count_replica # -Ndb_api_scan_batch_count_session # -Ndb_api_scan_batch_count_slave # -Ndb_api_table_scan_count # -Ndb_api_table_scan_count_replica # -Ndb_api_table_scan_count_session # -Ndb_api_table_scan_count_slave # -Ndb_api_trans_abort_count # -Ndb_api_trans_abort_count_replica # -Ndb_api_trans_abort_count_session # -Ndb_api_trans_abort_count_slave # -Ndb_api_trans_close_count # -Ndb_api_trans_close_count_replica # -Ndb_api_trans_close_count_session # -Ndb_api_trans_close_count_slave # -Ndb_api_trans_commit_count # -Ndb_api_trans_commit_count_replica # -Ndb_api_trans_commit_count_session # -Ndb_api_trans_commit_count_slave # -Ndb_api_trans_local_read_row_count # -Ndb_api_trans_local_read_row_count_replica # -Ndb_api_trans_local_read_row_count_session # -Ndb_api_trans_local_read_row_count_slave # -Ndb_api_trans_start_count # -Ndb_api_trans_start_count_replica # -Ndb_api_trans_start_count_session # -Ndb_api_trans_start_count_slave # -Ndb_api_uk_op_count # -Ndb_api_uk_op_count_replica # -Ndb_api_uk_op_count_session # -Ndb_api_uk_op_count_slave # -Ndb_api_wait_exec_complete_count # -Ndb_api_wait_exec_complete_count_replica # -Ndb_api_wait_exec_complete_count_session # -Ndb_api_wait_exec_complete_count_slave # -Ndb_api_wait_meta_request_count # -Ndb_api_wait_meta_request_count_replica # -Ndb_api_wait_meta_request_count_session # -Ndb_api_wait_meta_request_count_slave # -Ndb_api_wait_nanos_count # -Ndb_api_wait_nanos_count_replica # -Ndb_api_wait_nanos_count_session # -Ndb_api_wait_nanos_count_slave # -Ndb_api_wait_scan_result_count # -Ndb_api_wait_scan_result_count_replica # -Ndb_api_wait_scan_result_count_session # -Ndb_api_wait_scan_result_count_slave # -Ndb_cluster_node_id # -Ndb_config_from_host # -Ndb_config_from_port # -Ndb_config_generation # -Ndb_conflict_epoch_delete_delete_count # -Ndb_conflict_fn_epoch # -Ndb_conflict_fn_epoch2 # -Ndb_conflict_fn_epoch2_trans # -Ndb_conflict_fn_epoch_trans # -Ndb_conflict_fn_max # -Ndb_conflict_fn_max_del_win # -Ndb_conflict_fn_max_del_win_ins # -Ndb_conflict_fn_max_ins # -Ndb_conflict_fn_old # -Ndb_conflict_last_conflict_epoch # -Ndb_conflict_last_stable_epoch # -Ndb_conflict_reflected_op_discard_count # -Ndb_conflict_reflected_op_prepare_count # -Ndb_conflict_refresh_op_count # -Ndb_conflict_trans_conflict_commit_count # -Ndb_conflict_trans_detect_iter_count # -Ndb_conflict_trans_reject_count # -Ndb_conflict_trans_row_conflict_count # -Ndb_conflict_trans_row_reject_count # -Ndb_connect_count # -Ndb_execute_count # -Ndb_fetch_table_stats # -Ndb_index_stat_cache_clean # -Ndb_index_stat_cache_query # -Ndb_index_stat_event_count # -Ndb_index_stat_status # -Ndb_last_commit_epoch_server # -Ndb_last_commit_epoch_session # -Ndb_log_purge_queue_size # -Ndb_log_purged_files # -Ndb_log_purged_rows # -Ndb_metadata_detected_count # -Ndb_metadata_excluded_count # -Ndb_metadata_synced_count # -Ndb_number_of_data_nodes # -Ndb_number_of_ready_data_nodes # -Ndb_pruned_scan_count # -Ndb_pushed_queries_defined # -Ndb_pushed_queries_dropped # -Ndb_pushed_queries_executed # -Ndb_pushed_reads # -Ndb_replica_max_replicated_epoch # -Ndb_scan_count # -Ndb_schema_locks_count # -Ndb_slave_max_replicated_epoch # -Ndb_sorted_scan_count # -Ndb_system_name # -Ndb_trans_hint_count_session # -SHOW GLOBAL VARIABLES WHERE Variable_name LIKE 'ndb\_%' and -Variable_name NOT LIKE 'ndb_dbg\_%'; -Variable_name Value -ndb_allow_copying_alter_table # -ndb_applier_allow_skip_epoch # -ndb_applier_conflict_role # -ndb_autoincrement_prefetch_sz # -ndb_batch_size # -ndb_blob_read_batch_bytes # -ndb_blob_write_batch_bytes # -ndb_clear_apply_status # -ndb_cluster_connection_pool # -ndb_cluster_connection_pool_nodeids # -ndb_connectstring # -ndb_data_node_neighbour # -ndb_default_column_format # -ndb_deferred_constraints # -ndb_distribution # -ndb_eventbuffer_free_percent # -ndb_eventbuffer_max_alloc # -ndb_extra_logging # -ndb_force_send # -ndb_fully_replicated # -ndb_index_stat_enable # -ndb_index_stat_option # -ndb_join_pushdown # -ndb_log_apply_status # -ndb_log_bin # -ndb_log_binlog_index # -ndb_log_empty_epochs # -ndb_log_empty_update # -ndb_log_exclusive_reads # -ndb_log_fail_terminate # -ndb_log_orig # -ndb_log_purge_rate # -ndb_log_transaction_compression # -ndb_log_transaction_compression_level_zstd # -ndb_log_transaction_dependency # -ndb_log_transaction_id # -ndb_log_update_as_write # -ndb_log_update_minimal # -ndb_log_updated_only # -ndb_metadata_check # -ndb_metadata_check_interval # -ndb_metadata_sync # -ndb_mgm_tls # -ndb_mgmd_host # -ndb_nodeid # -ndb_optimization_delay # -ndb_optimized_node_selection # -ndb_read_backup # -ndb_recv_thread_activation_threshold # -ndb_recv_thread_cpu_mask # -ndb_replica_batch_size # -ndb_replica_blob_write_batch_bytes # -ndb_report_thresh_binlog_epoch_slip # -ndb_report_thresh_binlog_mem_usage # -ndb_row_checksum # -ndb_schema_dist_lock_wait_timeout # -ndb_schema_dist_timeout # -ndb_schema_dist_upgrade_allowed # -ndb_show_foreign_key_mock_tables # -ndb_slave_conflict_role # -ndb_table_no_logging # -ndb_table_temporary # -ndb_tls_search_path # -ndb_use_copying_alter_table # -ndb_use_exact_count # -ndb_use_transactions # -ndb_version # -ndb_version_string # -ndb_wait_connected # -ndb_wait_setup # CREATE TABLE t1 ( pk1 INT NOT NULL PRIMARY KEY, attr1 INT NOT NULL, @@ -1138,4 +920,46 @@ ENGINE=ndb; show warnings; Level Code Message drop table t1; +select node_id, remote_node_id, type +from ndbinfo.transporter_details +order by node_id, remote_node_id; +node_id remote_node_id type +1 2 TCP +1 2 TCP +1 2 TCP +1 2 TCP +1 3 TCP +1 16 TCP +1 32 TCP +1 48 TCP +1 49 TCP +1 63 TCP +1 127 TCP +1 192 TCP +1 228 TCP +1 229 TCP +1 230 TCP +1 231 TCP +1 232 TCP +1 233 TCP +1 255 TCP +2 1 TCP +2 1 TCP +2 1 TCP +2 1 TCP +2 3 TCP +2 16 TCP +2 32 TCP +2 48 TCP +2 49 TCP +2 63 TCP +2 127 TCP +2 192 TCP +2 228 TCP +2 229 TCP +2 230 TCP +2 231 TCP +2 232 TCP +2 233 TCP +2 255 TCP DATABASE QUOTA DROP successfully executed diff --git a/storage/ndb/compile-cluster b/storage/ndb/compile-cluster index 1a54459cecbc..4837c40a6cdb 100755 --- a/storage/ndb/compile-cluster +++ b/storage/ndb/compile-cluster @@ -310,10 +310,7 @@ if(! -d "$opt_srcdir/libmysqld") if ($opt_with_zlib_dir ne "bundled"); push(@args, "-DWITH_ZLIB=$opt_with_zlib_dir"); } - if($opt_with_ssl) - { - push(@args, "-DWITH_SSL=".($opt_with_ssl ? "yes" : "wolfssl")); - } + push(@args, "-DWITH_SSL=/usr/local/ssl"); if ($opt_localstatedir) { push(@args, "-DMYSQL_DATADIR=$opt_localstatedir"); diff --git a/storage/ndb/include/kernel/ndb_limits.h b/storage/ndb/include/kernel/ndb_limits.h index 9f94eb3b729e..a80e61af97cb 100644 --- a/storage/ndb/include/kernel/ndb_limits.h +++ b/storage/ndb/include/kernel/ndb_limits.h @@ -195,11 +195,13 @@ * API can order a multiple of this number of records at a time since * fragments can be scanned in parallel. */ -#define MAX_PARALLEL_OP_PER_SCAN 992 +#define MAX_PARALLEL_OP_PER_SCAN_WITH_LOCK 992 +#define MAX_PARALLEL_OP_PER_SCAN_SPJ 4000 +#define MAX_PARALLEL_OP_PER_SCAN_RC 4000 /* * The default batch size. Configurable parameter. */ -#define DEF_BATCH_SIZE 256 +#define DEF_BATCH_SIZE 384 /* * When calculating the number of records sent from LQH in each batch * one uses SCAN_BATCH_SIZE divided by the expected size of signals @@ -209,14 +211,14 @@ * signals. * This parameter is configurable, this is the default value. */ -#define SCAN_BATCH_SIZE 16384 +#define SCAN_BATCH_SIZE 32768 /* * To protect the NDB API from overload we also define a maximum total * batch size from all nodes. This parameter should most likely be * configurable, or dependent on sendBufferSize. * This parameter is configurable, this is the default value. */ -#define MAX_SCAN_BATCH_SIZE 262144 +#define MAX_SCAN_BATCH_SIZE 524288 /* * Maximum number of Parallel Scan queries on one hash index fragment */ diff --git a/storage/ndb/include/ndbapi/NdbReceiver.hpp b/storage/ndb/include/ndbapi/NdbReceiver.hpp index 3d6fd6dd8568..7f9f7bf548ac 100644 --- a/storage/ndb/include/ndbapi/NdbReceiver.hpp +++ b/storage/ndb/include/ndbapi/NdbReceiver.hpp @@ -128,11 +128,16 @@ class NdbReceiver { void getValues(const NdbRecord *, char *); void prepareSend(); - static void calculate_batch_size(const NdbImpl &, Uint32 parallelism, - Uint32 &batch_size, Uint32 &batch_byte_size); - - void calculate_batch_size(Uint32 parallelism, Uint32 &batch_size, - Uint32 &batch_byte_size) const; + static void calculate_batch_size(const NdbImpl &, + Uint32 parallelism, + Uint32 &batch_size, + Uint32 &batch_byte_size, + Uint32 def_max_batch_size); + + void calculate_batch_size(Uint32 parallelism, + Uint32 &batch_size, + Uint32 &batch_byte_size, + Uint32 def_max_batch_size) const; /** * Calculate size of result buffer which has to be diff --git a/storage/ndb/plugin/ha_ndbcluster.cc b/storage/ndb/plugin/ha_ndbcluster.cc index 8732df1853b9..0ff9ac3b60d8 100644 --- a/storage/ndb/plugin/ha_ndbcluster.cc +++ b/storage/ndb/plugin/ha_ndbcluster.cc @@ -9009,7 +9009,7 @@ int ha_ndbcluster::get_old_table_comment_items(THD *thd, if (mod_fully_replicated->m_found) comment_items_shown[FULLY_REPLICATED] = true; if (mod_frags->m_found) comment_items_shown[PARTITION_BALANCE] = true; - if (mod_ttl) { + if (mod_ttl->m_found) { comment_items_shown[TTL] = true; } return 0; diff --git a/storage/ndb/src/CMakeLists.txt b/storage/ndb/src/CMakeLists.txt index b9983fd7f835..f7feeb03c767 100644 --- a/storage/ndb/src/CMakeLists.txt +++ b/storage/ndb/src/CMakeLists.txt @@ -40,7 +40,9 @@ ENDFOREACH() ADD_SUBDIRECTORY(common) ADD_SUBDIRECTORY(mgmapi) ADD_SUBDIRECTORY(ndbapi) -ADD_SUBDIRECTORY(ronsql) +IF(WITH_RDRS) + ADD_SUBDIRECTORY(ronsql) +ENDIF() # # Build static ndbclient library diff --git a/storage/ndb/src/common/mgmcommon/ConfigInfo.cpp b/storage/ndb/src/common/mgmcommon/ConfigInfo.cpp index fc1e3e9dba95..2cd8d7baf879 100644 --- a/storage/ndb/src/common/mgmcommon/ConfigInfo.cpp +++ b/storage/ndb/src/common/mgmcommon/ConfigInfo.cpp @@ -539,7 +539,7 @@ const ConfigInfo::ParamInfo ConfigInfo::m_ParamInfo[] = { {CFG_DB_BATCH_SIZE, "BatchSizePerLocalScan", DB_TOKEN, "Used to calculate the number of lock records for scan with hold lock", ConfigInfo::CI_USED, false, ConfigInfo::CI_INT, STR_VALUE(DEF_BATCH_SIZE), - "1", STR_VALUE(MAX_PARALLEL_OP_PER_SCAN)}, + "1", STR_VALUE(MAX_PARALLEL_OP_PER_SCAN_RC)}, {CFG_DB_NO_TRANSACTIONS, "MaxNoOfConcurrentTransactions", DB_TOKEN, "Max number of transaction executing concurrently on the " DB_TOKEN_PRINT @@ -1651,7 +1651,7 @@ const ConfigInfo::ParamInfo ConfigInfo::m_ParamInfo[] = { {CFG_BATCH_SIZE, "BatchSize", "API", "The default batch size in number of records", ConfigInfo::CI_USED, false, ConfigInfo::CI_INT, STR_VALUE(DEF_BATCH_SIZE), "1", - STR_VALUE(MAX_PARALLEL_OP_PER_SCAN)}, + STR_VALUE(MAX_PARALLEL_OP_PER_SCAN_RC)}, {KEY_INTERNAL, "ConnectionMap", "API", "Specifies which DB nodes to connect", ConfigInfo::CI_USED, false, diff --git a/storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp b/storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp index 3339d965ee33..1eba3decc4df 100644 --- a/storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp +++ b/storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp @@ -647,8 +647,9 @@ class Dblqh : public SimulatedBlock { }; enum ScanType { ST_IDLE = 0, SCAN = 1, COPY = 2 }; - /* A single scan of each fragment can have MAX_PARALLEL_OP_PER_SCAN - * read operations in progress at one time + /* A single scan of each fragment can have + * MAX_PARALLEL_OP_PER_SCAN_WITH_LOCK read operations in progress + * at one time if we are reading with locks. * We must store ACC ptrs for each read operation. They are stored * in SegmentedSections linked in the array below. * The main oddity is that the first element of scan_acc_op_ptr is @@ -656,9 +657,9 @@ class Dblqh : public SimulatedBlock { * ACC ptrs. */ static constexpr Uint32 MaxScanAccSegments = - ((MAX_PARALLEL_OP_PER_SCAN + SectionSegment::DataLength - 1) / - SectionSegment::DataLength) + - 1; + ((MAX_PARALLEL_OP_PER_SCAN_RC + + SectionSegment::DataLength - 1) / + SectionSegment::DataLength) + 1; UintR scan_acc_op_ptr[MaxScanAccSegments]; Uint32 scan_acc_index; @@ -673,6 +674,7 @@ class Dblqh : public SimulatedBlock { Uint32 m_curr_batch_size_bytes; Uint32 m_exec_direct_batch_size_words; + Uint32 m_def_max_batch_size; bool check_scan_batch_completed(bool print = false) const; @@ -2802,11 +2804,13 @@ class Dblqh : public SimulatedBlock { //copyCountWords must be set before used reqRef(Uint32(~0)), - reqBlockref(Uint32(~0)) + reqBlockref(Uint32(~0)), //m_corrFactorLo must be set before used //m_corrFactorHi must be set before used //scanKeyInfoPos only used when m_flags has OP_SCANKEYINFOPOSSAVED set //m_nr_delete only used in Copy fragment, set before used + original_operation(0xFF), + ttl_ignore(0) { m_dealloc_data.m_unused = RNIL; #ifdef DEBUG_USAGE_COUNT diff --git a/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp b/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp index 44d5a393e8e0..be6e51d6a348 100644 --- a/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp +++ b/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp @@ -14123,6 +14123,8 @@ void Dblqh::releaseTcrec(Signal *signal, TcConnectionrecPtr locTcConnectptr) { ndbrequire(pre_usageCountW > 0); } } + locTcConnectptr.p->original_operation = 0xFF; + locTcConnectptr.p->ttl_ignore = 0; Dblqh *lqh = m_curr_lqh; if (likely(locTcConnectptr.i < lqh->ctcConnectReserved)) { @@ -17335,7 +17337,8 @@ void Dblqh::init_acc_ptr_list(ScanRecord *scanP) { scanP->scan_acc_index = 0; } Uint32 Dblqh::get_acc_ptr_from_scan_record(ScanRecord *scanP, Uint32 index, bool crash_flag) { Uint32 *acc_ptr; - if (!((index < MAX_PARALLEL_OP_PER_SCAN) && index < scanP->scan_acc_index)) { + if (!((index < MAX_PARALLEL_OP_PER_SCAN_WITH_LOCK) && + index < scanP->scan_acc_index)) { ndbrequire(crash_flag); return RNIL; } @@ -17347,7 +17350,7 @@ void Dblqh::set_acc_ptr_in_scan_record(ScanRecord *scanP, Uint32 index, Uint32 acc) { Uint32 *acc_ptr; ndbrequire((index == 0 || scanP->scan_acc_index == index) && - (index < MAX_PARALLEL_OP_PER_SCAN)); + (index < MAX_PARALLEL_OP_PER_SCAN_WITH_LOCK)); scanP->scan_acc_index = index + 1; i_get_acc_ptr(scanP, acc_ptr, index); *acc_ptr = acc; @@ -17800,7 +17803,10 @@ void Dblqh::execSCAN_FRAGREQ(Signal *signal) { */ ndbrequire(scanLockMode == 0 || keyinfo); - ndbrequire(max_rows > 0 && max_rows <= MAX_PARALLEL_OP_PER_SCAN); + Uint32 def_max_batch_size = (scanLockMode == 0) ? + MAX_PARALLEL_OP_PER_SCAN_RC : + MAX_PARALLEL_OP_PER_SCAN_WITH_LOCK; + ndbrequire(max_rows > 0 && max_rows <= def_max_batch_size); // Verify scan type vs table type (both sides are boolean) if (unlikely(rangeScan != @@ -17822,6 +17828,7 @@ void Dblqh::execSCAN_FRAGREQ(Signal *signal) { releaseSections(handle); goto error_handler; } + scanptr.p->m_def_max_batch_size = def_max_batch_size; initScanTc(scanFragReq, transid1, transid2, fragId, ZNIL, senderBlockRef, tcConnectptr); regTcPtr->opExec = (1 ^ ScanFragReq::getNotInterpretedFlag(reqinfo)); @@ -18580,7 +18587,8 @@ void Dblqh::nextScanConfScanLab(Signal *signal, ScanRecord *const scanPtr, scanPtr->m_row_id.m_page_idx, gci); readLength = 3; } - ndbrequire(scanPtr->m_curr_batch_size_rows < MAX_PARALLEL_OP_PER_SCAN); + ndbrequire(scanPtr->m_curr_batch_size_rows < + scanPtr->m_def_max_batch_size); scanPtr->m_exec_direct_batch_size_words += readLength; scanPtr->m_curr_batch_size_bytes += readLength * sizeof(Uint32); scanPtr->m_curr_batch_size_rows++; @@ -18963,9 +18971,9 @@ void Dblqh::scanTupkeyConfLab(Signal* signal, * Moz * In aggregation mode, since we don't follow batch strategy 100%, * in the situation which has small group, m_curr_batch_size_rows - * could be bigger than MAX_PARALLEL_OP_PER_SCAN + * could be bigger than MAX_PARALLEL_OP_PER_SCAN_RC */ - ndbrequire(scanPtr->m_curr_batch_size_rows < MAX_PARALLEL_OP_PER_SCAN); + ndbrequire(scanPtr->m_curr_batch_size_rows < scanPtr->m_def_max_batch_size); } scanPtr->m_exec_direct_batch_size_words += read_len; scanPtr->m_curr_batch_size_bytes += read_len * sizeof(Uint32); @@ -19924,7 +19932,7 @@ void Dblqh::init_release_scanrec(ScanRecord *scanPtr) { * ------------------------------------------------------------------------ */ Uint32 Dblqh::sendKeyinfo20(Signal *signal, ScanRecord *scanP, TcConnectionrec *tcConP) { - ndbrequire(scanP->m_curr_batch_size_rows < MAX_PARALLEL_OP_PER_SCAN); + ndbrequire(scanP->m_curr_batch_size_rows < scanP->m_def_max_batch_size); KeyInfo20 *keyInfo = (KeyInfo20 *)&signal->theData[0]; /** diff --git a/storage/ndb/src/kernel/blocks/dbspj/DbspjMain.cpp b/storage/ndb/src/kernel/blocks/dbspj/DbspjMain.cpp index a03b4e08d882..ca99d2dcde5e 100644 --- a/storage/ndb/src/kernel/blocks/dbspj/DbspjMain.cpp +++ b/storage/ndb/src/kernel/blocks/dbspj/DbspjMain.cpp @@ -6892,17 +6892,17 @@ Uint32 Dbspj::scanFrag_parallelism(Ptr requestPtr, /** * Parallelism must be increased if we otherwise would be limited - * by the MAX_PARALLEL_OP_PER_SCAN limitation in the SCAN_FRAGREQs + * by the MAX_PARALLEL_OP_PER_SCAN_SPJ limitation in the SCAN_FRAGREQs */ const ScanFragReq *req = reinterpret_cast(data.m_scanFragReq); const Uint32 availableBatchRows = req->batch_size_rows - data.m_totalRows; ndbrequire(availableBatchRows >= batchSizeRows); - if ((availableBatchRows / parallelism) > MAX_PARALLEL_OP_PER_SCAN) { + if ((availableBatchRows / parallelism) > MAX_PARALLEL_OP_PER_SCAN_SPJ) { jam(); - parallelism = MIN((availableBatchRows + MAX_PARALLEL_OP_PER_SCAN - 1) / - MAX_PARALLEL_OP_PER_SCAN, + parallelism = MIN((availableBatchRows + MAX_PARALLEL_OP_PER_SCAN_SPJ - 1) / + MAX_PARALLEL_OP_PER_SCAN_SPJ, data.m_frags_not_started); } @@ -7008,9 +7008,10 @@ void Dbspj::scanFrag_send(Signal *signal, Ptr requestPtr, treeNodePtr, availableBatchBytes, availableBatchRows); data.m_parallelism = scanFrag_parallelism(requestPtr, treeNodePtr, batchRows); - // Cap batchSize-rows to avoid exceeding MAX_PARALLEL_OP_PER_SCAN + // Cap batchSize-rows to avoid exceeding MAX_PARALLEL_OP_PER_SCAN_SPJ const Uint32 bs_rows = - MIN(availableBatchRows / data.m_parallelism, MAX_PARALLEL_OP_PER_SCAN); + MIN(availableBatchRows / data.m_parallelism, + MAX_PARALLEL_OP_PER_SCAN_SPJ); const Uint32 bs_bytes = availableBatchBytes / data.m_parallelism; ndbassert(bs_rows > 0); ndbassert(bs_bytes > 0); @@ -7081,7 +7082,7 @@ Uint32 Dbspj::scanFrag_send(Signal *signal, Ptr requestPtr, // req->variableData[0] // set below req->variableData[1] = requestPtr.p->m_rootResultData; req->batch_size_bytes = bs_bytes; - req->batch_size_rows = MIN(bs_rows, MAX_PARALLEL_OP_PER_SCAN); + req->batch_size_rows = MIN(bs_rows, MAX_PARALLEL_OP_PER_SCAN_SPJ); /** * A SORTED_ORDER scan need to fetch one row at a time from the treeNode @@ -8032,7 +8033,8 @@ void Dbspj::scanFrag_execSCAN_NEXTREQ(Signal *signal, Ptr requestPtr, } Uint32 bs_rows = - MIN(org->batch_size_rows / data.m_parallelism, MAX_PARALLEL_OP_PER_SCAN); + MIN(org->batch_size_rows / data.m_parallelism, + MAX_PARALLEL_OP_PER_SCAN_SPJ); const Uint32 bs_bytes = org->batch_size_bytes / data.m_parallelism; ndbassert(bs_rows > 0); diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupPageMap.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupPageMap.cpp index 8157b94f1478..b1ccac602511 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupPageMap.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupPageMap.cpp @@ -494,6 +494,11 @@ Uint32 Dbtup::insert_new_page_into_page_map(EmulatedJamBuffer *jamBuf, jam(); *prev_ptr = FREE_PAGE_BIT | LAST_LCP_FREE_BIT; } + /* Decrement number of pages used in database */ + m_ldm_instance_used->c_lqh->update_memory_usage(regFragPtr->fragTableId, + Int32(-4 * noOfPagesAllocated), + __LINE__, + pagePtr.i); returnCommonArea(pagePtr.i, noOfPagesAllocated); return RNIL; } diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp index 32f88d249529..909f1f060535 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp @@ -3009,6 +3009,10 @@ void Dbtup::handle_lcp_drop_change_page(Fragrecord *fragPtrP, returnCommonArea(pagePtr.i, 1); return; } + m_ldm_instance_used->c_lqh->update_memory_usage(fragPtrP->fragTableId, + Int32(-4), + __LINE__, + pagePtr.i); m_ctx.m_mm.lock(); m_ctx.m_mm.release_page(RT_DBTUP_PAGE, pagePtr.i, true); Uint32 words = 6 + ((found_idx_count + 1) / 2); diff --git a/storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp b/storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp index 0026988a2ca8..87b8dda5ad02 100644 --- a/storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp +++ b/storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp @@ -112,7 +112,7 @@ class Dbtux : public SimulatedBlock { static constexpr Uint32 MaxTreeNodeSize = MAX_TTREE_NODE_SIZE; static constexpr Uint32 MaxPrefSize = MAX_TTREE_PREF_SIZE; static constexpr Uint32 ScanBoundSegmentSize = 7; - static constexpr Uint32 MaxAccLockOps = MAX_PARALLEL_OP_PER_SCAN; + static constexpr Uint32 MaxAccLockOps = MAX_PARALLEL_OP_PER_SCAN_WITH_LOCK; static constexpr Uint32 MaxTreeDepth = 32; // strict #ifdef VM_TRACE // for TuxCtx::c_debugBuffer diff --git a/storage/ndb/src/ndbapi/NdbQueryOperation.cpp b/storage/ndb/src/ndbapi/NdbQueryOperation.cpp index 88c7b99b1b84..76147ff86e4f 100644 --- a/storage/ndb/src/ndbapi/NdbQueryOperation.cpp +++ b/storage/ndb/src/ndbapi/NdbQueryOperation.cpp @@ -4527,7 +4527,9 @@ Uint32 NdbQueryOperationImpl ::calculateBatchedRows( getRoot().m_parallelism == Parallelism_max ? rootFragments : getRoot().m_parallelism, - maxBatchRows, batchByteSize); + maxBatchRows, + batchByteSize, + MAX_PARALLEL_OP_PER_SCAN_SPJ); assert(maxBatchRows > 0); assert(maxBatchRows <= batchByteSize); @@ -5391,8 +5393,11 @@ Uint32 NdbQueryOperationImpl::getMaxBatchBytes() const { if (m_operationDef.isScanOperation()) { const Ndb *const ndb = getQuery().getNdbTransaction().getNdb(); const Uint32 parallelism = rootFragments; - NdbReceiver::calculate_batch_size(*ndb->theImpl, parallelism, batchRows, - batchByteSize); + NdbReceiver::calculate_batch_size(*ndb->theImpl, + parallelism, + batchRows, + batchByteSize, + MAX_PARALLEL_OP_PER_SCAN_SPJ); assert(batchRows == getMaxBatchRows()); /** diff --git a/storage/ndb/src/ndbapi/NdbReceiver.cpp b/storage/ndb/src/ndbapi/NdbReceiver.cpp index 3a41e3c269f4..760a786c8400 100644 --- a/storage/ndb/src/ndbapi/NdbReceiver.cpp +++ b/storage/ndb/src/ndbapi/NdbReceiver.cpp @@ -501,8 +501,10 @@ void NdbReceiver::prepareReceive(NdbReceiverBuffer *buffer) { */ // static void NdbReceiver::calculate_batch_size(const NdbImpl &theImpl, - Uint32 parallelism, Uint32 &batch_size, - Uint32 &batch_byte_size) { + Uint32 parallelism, + Uint32 &batch_size, + Uint32 &batch_byte_size, + Uint32 def_max_batch_size) { const NdbApiConfig &cfg = theImpl.get_ndbapi_config_parameters(); const Uint32 max_scan_batch_size = cfg.m_scan_batch_size; const Uint32 max_batch_byte_size = cfg.m_batch_byte_size; @@ -516,18 +518,23 @@ void NdbReceiver::calculate_batch_size(const NdbImpl &theImpl, if (batch_size == 0 || batch_size > max_batch_size) { batch_size = max_batch_size; } - if (unlikely(batch_size > MAX_PARALLEL_OP_PER_SCAN)) { - batch_size = MAX_PARALLEL_OP_PER_SCAN; + if (unlikely(batch_size > def_max_batch_size)) { + batch_size = def_max_batch_size; } if (unlikely(batch_size > batch_byte_size)) { batch_size = batch_byte_size; } } -void NdbReceiver::calculate_batch_size(Uint32 parallelism, Uint32 &batch_size, - Uint32 &batch_byte_size) const { - calculate_batch_size(*m_ndb->theImpl, parallelism, batch_size, - batch_byte_size); +void NdbReceiver::calculate_batch_size(Uint32 parallelism, + Uint32 &batch_size, + Uint32 &batch_byte_size, + Uint32 def_max_batch_size) const { + calculate_batch_size(*m_ndb->theImpl, + parallelism, + batch_size, + batch_byte_size, + def_max_batch_size); } // static diff --git a/storage/ndb/src/ndbapi/NdbScanOperation.cpp b/storage/ndb/src/ndbapi/NdbScanOperation.cpp index a091dd10b5ff..6b0ec02c735a 100644 --- a/storage/ndb/src/ndbapi/NdbScanOperation.cpp +++ b/storage/ndb/src/ndbapi/NdbScanOperation.cpp @@ -467,7 +467,8 @@ inline int NdbScanOperation::scanImpl( * Zart * TTL */ - if (options->optionsPresent & ScanOptions::SO_TTL_IGNORE) { + if (options && + options->optionsPresent & ScanOptions::SO_TTL_IGNORE) { m_flags |= OF_TTL_IGNORE; } @@ -2187,13 +2188,18 @@ int NdbScanOperation::prepareSendScan(Uint32 /*aTC_ConnectPtr*/, /* All scans use NdbRecord internally */ assert(theStatus == UseNdbRecord); + Uint32 def_max_batch_size = ScanTabReq::getLockMode(reqInfo) == 0 ? + MAX_PARALLEL_OP_PER_SCAN_RC : MAX_PARALLEL_OP_PER_SCAN_WITH_LOCK; /** * The number of records sent by each LQH is calculated and the kernel * is informed of this number by updating the SCAN_TABREQ signal */ Uint32 batch_size = req->first_batch_size; // Possibly user specified Uint32 batch_byte_size = 0; - theReceiver.calculate_batch_size(theParallelism, batch_size, batch_byte_size); + theReceiver.calculate_batch_size(theParallelism, + batch_size, + batch_byte_size, + def_max_batch_size); /** * Calculate memory req. for the NdbReceiverBuffer and its row buffer: diff --git a/storage/ndb/test/run-test/main.cpp b/storage/ndb/test/run-test/main.cpp index 0ffc820c0924..2e5f60ed1c1c 100644 --- a/storage/ndb/test/run-test/main.cpp +++ b/storage/ndb/test/run-test/main.cpp @@ -1018,7 +1018,9 @@ void test_case_results(TestResult *test_result, const atrt_testcase &testcase) { res_dir.assfmt("result.%d", testcase.test_no); remove_dir(res_dir.c_str(), true); - if (testcase.m_report || test_result->result != ERR_OK) { + if (testcase.m_report || + (test_result->result != ERR_OK && + test_result->result != ERR_SKIPPED)) { if (rename("result", res_dir.c_str()) != 0) { g_logger.critical("Failed to rename %s as %s", "result", res_dir.c_str()); remove_dir("result", true);