Merge pull request #5945 from multiversx/rc/v1.6.next1

Rc/v1.6.next1

api/groups/nodeGroup.go

@@ -28,6 +28,7 @@ const (
 	bootstrapStatusPath       = "/bootstrapstatus"
 	connectedPeersRatingsPath = "/connected-peers-ratings"
 	managedKeys               = "/managed-keys"
+	loadedKeys                = "/loaded-keys"
 	managedKeysCount          = "/managed-keys/count"
 	eligibleManagedKeys       = "/managed-keys/eligible"
 	waitingManagedKeys        = "/managed-keys/waiting"
@@ -43,6 +44,7 @@ type nodeFacadeHandler interface {
 	GetConnectedPeersRatingsOnMainNetwork() (string, error)
 	GetManagedKeysCount() int
 	GetManagedKeys() []string
+	GetLoadedKeys() []string
 	GetEligibleManagedKeys() ([]string, error)
 	GetWaitingManagedKeys() ([]string, error)
 	IsInterfaceNil() bool
@@ -127,6 +129,11 @@ func NewNodeGroup(facade nodeFacadeHandler) (*nodeGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ng.managedKeys,
 		},
+		{
+			Path:    loadedKeys,
+			Method:  http.MethodGet,
+			Handler: ng.loadedKeys,
+		},
 		{
 			Path:    eligibleManagedKeys,
 			Method:  http.MethodGet,
@@ -404,6 +411,19 @@ func (ng *nodeGroup) managedKeys(c *gin.Context) {
 	)
 }
 
+// loadedKeys returns all keys loaded by the current node
+func (ng *nodeGroup) loadedKeys(c *gin.Context) {
+	keys := ng.getFacade().GetLoadedKeys()
+	c.JSON(
+		http.StatusOK,
+		shared.GenericAPIResponse{
+			Data:  gin.H{"loadedKeys": keys},
+			Error: "",
+			Code:  shared.ReturnCodeSuccess,
+		},
+	)
+}
+
 // managedKeysEligible returns the node's eligible managed keys
 func (ng *nodeGroup) managedKeysEligible(c *gin.Context) {
 	keys, err := ng.getFacade().GetEligibleManagedKeys()

api/groups/nodeGroup_test.go

@@ -81,6 +81,13 @@ type managedKeysResponse struct {
 	generalResponse
 }
 
+type loadedKeysResponse struct {
+	Data struct {
+		LoadedKeys []string `json:"loadedKeys"`
+	} `json:"data"`
+	generalResponse
+}
+
 type managedEligibleKeysResponse struct {
 	Data struct {
 		Keys []string `json:"eligibleKeys"`
@@ -733,6 +740,36 @@ func TestNodeGroup_ManagedKeys(t *testing.T) {
 	assert.Equal(t, providedKeys, response.Data.ManagedKeys)
 }
 
+func TestNodeGroup_LoadedKeys(t *testing.T) {
+	t.Parallel()
+
+	providedKeys := []string{
+		"pk1",
+		"pk2",
+	}
+	facade := mock.FacadeStub{
+		GetLoadedKeysCalled: func() []string {
+			return providedKeys
+		},
+	}
+
+	nodeGroup, err := groups.NewNodeGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(nodeGroup, "node", getNodeRoutesConfig())
+
+	req, _ := http.NewRequest("GET", "/node/loaded-keys", nil)
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	response := &loadedKeysResponse{}
+	loadResponse(resp.Body, response)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+	assert.Equal(t, "", response.Error)
+	assert.Equal(t, providedKeys, response.Data.LoadedKeys)
+}
+
 func TestNodeGroup_ManagedKeysEligible(t *testing.T) {
 	t.Parallel()
 
@@ -960,6 +997,7 @@ func getNodeRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/connected-peers-ratings", Open: true},
 					{Name: "/managed-keys/count", Open: true},
 					{Name: "/managed-keys", Open: true},
+					{Name: "/loaded-keys", Open: true},
 					{Name: "/managed-keys/eligible", Open: true},
 					{Name: "/managed-keys/waiting", Open: true},
 				},

api/mock/facadeStub.go

@@ -91,6 +91,7 @@ type FacadeStub struct {
 	IsDataTrieMigratedCalled                    func(address string, options api.AccountQueryOptions) (bool, error)
 	GetManagedKeysCountCalled                   func() int
 	GetManagedKeysCalled                        func() []string
+	GetLoadedKeysCalled                         func() []string
 	GetEligibleManagedKeysCalled                func() ([]string, error)
 	GetWaitingManagedKeysCalled                 func() ([]string, error)
 }
@@ -594,6 +595,14 @@ func (f *FacadeStub) GetManagedKeys() []string {
 	return make([]string, 0)
 }
 
+// GetLoadedKeys -
+func (f *FacadeStub) GetLoadedKeys() []string {
+	if f.GetLoadedKeysCalled != nil {
+		return f.GetLoadedKeysCalled()
+	}
+	return make([]string, 0)
+}
+
 // GetEligibleManagedKeys -
 func (f *FacadeStub) GetEligibleManagedKeys() ([]string, error) {
 	if f.GetEligibleManagedKeysCalled != nil {

api/shared/interface.go

@@ -130,6 +130,7 @@ type FacadeHandler interface {
 	IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error)
 	GetManagedKeysCount() int
 	GetManagedKeys() []string
+	GetLoadedKeys() []string
 	GetEligibleManagedKeys() ([]string, error)
 	GetWaitingManagedKeys() ([]string, error)
 	IsInterfaceNil() bool

cmd/node/config/api.toml

@@ -43,6 +43,9 @@
         # /node/managed-keys will return the keys managed by the node
         { Name = "/managed-keys", Open = true },
 
+        # /node/loaded-keys will return the keys loaded by the node
+        { Name = "/loaded-keys", Open = true },
+
         # /node/managed-keys/count will return the number of keys managed by the node
         { Name = "/managed-keys/count", Open = true },
 

cmd/node/config/config.toml

@@ -519,6 +519,7 @@
 [Antiflood]
     Enabled = true
     NumConcurrentResolverJobs = 50
+    NumConcurrentResolvingTrieNodesJobs = 3
     [Antiflood.FastReacting]
         IntervalInSeconds = 1
         ReservedPercent   = 20.0

cmd/node/config/fullArchiveP2P.toml

@@ -23,10 +23,11 @@
         [Node.Transports.TCP]
             ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address
             PreventPortReuse = false
-        [Node.ResourceLimiter]
-            Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
-            ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
-            ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
+
+    [Node.ResourceLimiter]
+        Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
+        ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
+        ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
 
 # P2P peer discovery section
 

cmd/node/config/p2p.toml

@@ -23,10 +23,11 @@
         [Node.Transports.TCP]
             ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address
             PreventPortReuse = false
-        [Node.ResourceLimiter]
-            Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
-            ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
-            ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
+
+    [Node.ResourceLimiter]
+        Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
+        ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
+        ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
 
 # P2P peer discovery section
 

cmd/node/flags.go

@@ -632,7 +632,8 @@ func applyCompatibleConfigs(log logger.Logger, configs *config.Configs) error {
 
 	isInHistoricalBalancesMode := operationmodes.SliceContainsElement(operationModes, operationmodes.OperationModeHistoricalBalances)
 	if isInHistoricalBalancesMode {
-		processHistoricalBalancesMode(log, configs)
+		// TODO move all operation modes settings in the common/operationmodes package and add tests
+		operationmodes.ProcessHistoricalBalancesMode(log, configs)
 	}
 
 	isInDbLookupExtensionMode := operationmodes.SliceContainsElement(operationModes, operationmodes.OperationModeDbLookupExtension)
@@ -648,28 +649,6 @@ func applyCompatibleConfigs(log logger.Logger, configs *config.Configs) error {
 	return nil
 }
 
-func processHistoricalBalancesMode(log logger.Logger, configs *config.Configs) {
-	configs.GeneralConfig.StoragePruning.Enabled = true
-	configs.GeneralConfig.StoragePruning.ValidatorCleanOldEpochsData = false
-	configs.GeneralConfig.StoragePruning.ObserverCleanOldEpochsData = false
-	configs.GeneralConfig.GeneralSettings.StartInEpochEnabled = false
-	configs.GeneralConfig.StoragePruning.AccountsTrieCleanOldEpochsData = false
-	configs.GeneralConfig.StateTriesConfig.AccountsStatePruningEnabled = false
-	configs.GeneralConfig.DbLookupExtensions.Enabled = true
-	configs.PreferencesConfig.Preferences.FullArchive = true
-
-	log.Warn("the node is in historical balances mode! Will auto-set some config values",
-		"StoragePruning.Enabled", configs.GeneralConfig.StoragePruning.Enabled,
-		"StoragePruning.ValidatorCleanOldEpochsData", configs.GeneralConfig.StoragePruning.ValidatorCleanOldEpochsData,
-		"StoragePruning.ObserverCleanOldEpochsData", configs.GeneralConfig.StoragePruning.ObserverCleanOldEpochsData,
-		"StoragePruning.AccountsTrieCleanOldEpochsData", configs.GeneralConfig.StoragePruning.AccountsTrieCleanOldEpochsData,
-		"GeneralSettings.StartInEpochEnabled", configs.GeneralConfig.GeneralSettings.StartInEpochEnabled,
-		"StateTriesConfig.AccountsStatePruningEnabled", configs.GeneralConfig.StateTriesConfig.AccountsStatePruningEnabled,
-		"DbLookupExtensions.Enabled", configs.GeneralConfig.DbLookupExtensions.Enabled,
-		"Preferences.FullArchive", configs.PreferencesConfig.Preferences.FullArchive,
-	)
-}
-
 func processDbLookupExtensionMode(log logger.Logger, configs *config.Configs) {
 	configs.GeneralConfig.DbLookupExtensions.Enabled = true
 	configs.GeneralConfig.StoragePruning.Enabled = true

cmd/seednode/config/p2p.toml

@@ -22,10 +22,11 @@
         [Node.Transports.TCP]
             ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address
             PreventPortReuse = true # seeder nodes will need to enable this option
-        [Node.ResourceLimiter]
-            Type = "default with manual scale"
-            ManualSystemMemoryInMB = 65536 # pretend that the host running the seeder has more RAM so it can handle more connections
-            ManualMaximumFD = 1048576
+
+    [Node.ResourceLimiter]
+        Type = "default with manual scale"
+        ManualSystemMemoryInMB = 65536 # pretend that the host running the seeder has more RAM so it can handle more connections
+        ManualMaximumFD = 1048576
 
 # P2P peer discovery section
 

common/interface.go

@@ -414,6 +414,7 @@ type ManagedPeersHolder interface {
 	IncrementRoundsWithoutReceivedMessages(pkBytes []byte)
 	ResetRoundsWithoutReceivedMessages(pkBytes []byte, pid core.PeerID)
 	GetManagedKeysByCurrentNode() map[string]crypto.PrivateKey
+	GetLoadedKeysByCurrentNode() [][]byte
 	IsKeyManagedByCurrentNode(pkBytes []byte) bool
 	IsKeyRegistered(pkBytes []byte) bool
 	IsPidManagedByCurrentNode(pid core.PeerID) bool
@@ -443,6 +444,7 @@ type StateSyncNotifierSubscriber interface {
 type ManagedPeersMonitor interface {
 	GetManagedKeysCount() int
 	GetManagedKeys() [][]byte
+	GetLoadedKeys() [][]byte
 	GetEligibleManagedKeys() ([][]byte, error)
 	GetWaitingManagedKeys() ([][]byte, error)
 	IsInterfaceNil() bool

common/operationmodes/historicalBalances.go

@@ -0,0 +1,41 @@
+package operationmodes
+
+import (
+	"github.com/multiversx/mx-chain-go/config"
+	logger "github.com/multiversx/mx-chain-logger-go"
+)
+
+// ProcessHistoricalBalancesMode will process the provided flags for the historical balances
+func ProcessHistoricalBalancesMode(log logger.Logger, configs *config.Configs) {
+	configs.GeneralConfig.StoragePruning.Enabled = true
+	configs.GeneralConfig.StoragePruning.ValidatorCleanOldEpochsData = false
+	configs.GeneralConfig.StoragePruning.ObserverCleanOldEpochsData = false
+	configs.GeneralConfig.GeneralSettings.StartInEpochEnabled = false
+	configs.GeneralConfig.StoragePruning.AccountsTrieCleanOldEpochsData = false
+	configs.GeneralConfig.StateTriesConfig.AccountsStatePruningEnabled = false
+	configs.GeneralConfig.DbLookupExtensions.Enabled = true
+	configs.PreferencesConfig.Preferences.FullArchive = true
+
+	log.Warn("the node is in historical balances mode! Will auto-set some config values",
+		"StoragePruning.Enabled", configs.GeneralConfig.StoragePruning.Enabled,
+		"StoragePruning.ValidatorCleanOldEpochsData", configs.GeneralConfig.StoragePruning.ValidatorCleanOldEpochsData,
+		"StoragePruning.ObserverCleanOldEpochsData", configs.GeneralConfig.StoragePruning.ObserverCleanOldEpochsData,
+		"StoragePruning.AccountsTrieCleanOldEpochsData", configs.GeneralConfig.StoragePruning.AccountsTrieCleanOldEpochsData,
+		"GeneralSettings.StartInEpochEnabled", configs.GeneralConfig.GeneralSettings.StartInEpochEnabled,
+		"StateTriesConfig.AccountsStatePruningEnabled", configs.GeneralConfig.StateTriesConfig.AccountsStatePruningEnabled,
+		"DbLookupExtensions.Enabled", configs.GeneralConfig.DbLookupExtensions.Enabled,
+		"Preferences.FullArchive", configs.PreferencesConfig.Preferences.FullArchive,
+	)
+}
+
+// IsInHistoricalBalancesMode returns true if the configuration provided denotes a historical balances mode
+func IsInHistoricalBalancesMode(configs *config.Configs) bool {
+	return configs.GeneralConfig.StoragePruning.Enabled &&
+		!configs.GeneralConfig.StoragePruning.ValidatorCleanOldEpochsData &&
+		!configs.GeneralConfig.StoragePruning.ObserverCleanOldEpochsData &&
+		!configs.GeneralConfig.GeneralSettings.StartInEpochEnabled &&
+		!configs.GeneralConfig.StoragePruning.AccountsTrieCleanOldEpochsData &&
+		!configs.GeneralConfig.StateTriesConfig.AccountsStatePruningEnabled &&
+		configs.GeneralConfig.DbLookupExtensions.Enabled &&
+		configs.PreferencesConfig.Preferences.FullArchive
+}