Embed the scope NAVs CSV. (#2050)

* [1760]: Embed the upgrade_files directory into the build.

* [1760]: Move the testnet scope navs csv into an umber directory since those are specific to this upgrade.

* [1760]: Use the embeded file for reading the navs. Move the reading of that file to after the log message stating what's being done. Tweak the names of some stuff to standardize on all-caps NAV (go's standard for acronyms).

* [1760]: Update the log messages from addScopeNAVsWithHeight to have more info. Enhance TestAddScopeNAVsWithHeight to hit more cases and check also have it check the logs.

* [1760]: Add changelog entry and fix a couple other entries.

CHANGELOG.md

@@ -48,7 +48,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * Update stargate queries for Attribute, Exchange, Marker, IBCRateLimit, Metadata, Msgfees, and Oracle modules [#1760](https://github.com/provenance-io/provenance/issues/1760).
 * Update stargate queries for Quarantine and Sanction modules [#2016](https://github.com/provenance-io/provenance/pull/2016).
 * Add the circuit breaker module [#2031](https://github.com/provenance-io/provenance/pull/2031).
-* Add upgrade handler to set scope net asset values and update block height for pio-testnet-1 [#2046](https://github.com/provenance-io/provenance/pull/2046)
+* Add upgrade handler to set scope net asset values and update block height for pio-testnet-1 [#2046](https://github.com/provenance-io/provenance/pull/2046), [#2050](https://github.com/provenance-io/provenance/pull/2050).
 
 ### Improvements
 
@@ -184,7 +184,6 @@ Ref: https://keepachangelog.com/en/1.0.0/
   * /cosmos.staking.v1beta1.Query/HistoricalInfo
   * /cosmos.staking.v1beta1.Query/Pool
 
-
 ### Client Breaking
 
 * The `provenanced query account` command has been removed. It is still available as `provenanced query auth account` [#1971](https://github.com/provenance-io/provenance/pull/1971).
@@ -235,8 +234,8 @@ Ref: https://keepachangelog.com/en/1.0.0/
 - Bump `cosmossdk.io/client/v2` from 2.0.0-beta.1 to 2.0.0-beta.2 ([#2042](https://github.com/provenance-io/provenance/pull/2042))
 - Bump `docker/build-push-action` from 5 to 6 ([#2039](https://github.com/provenance-io/provenance/pull/2039))
 - Bump `github.com/cosmos/ibc-go/v8` from 8.2.1 to 8.3.2 ([#2043](https://github.com/provenance-io/provenance/pull/2043))
-- Bump wasmd to `v0.51.0` [#2045](https://github.com/provenance-io/provenance/pull/2045)
-- Bump wasmvm to `v2.0.1` [#2045](https://github.com/provenance-io/provenance/pull/2045)
+- Bump wasmd to `v0.51.0` ([#2045](https://github.com/provenance-io/provenance/pull/2045))
+- Bump wasmvm to `v2.0.1` ([#2045](https://github.com/provenance-io/provenance/pull/2045))
 
 ---
 

app/embed.go

@@ -0,0 +1,6 @@
+package app
+
+import "embed"
+
+//go:embed upgrade_files
+var UpgradeFiles embed.FS

app/scope_navs_updater.go

@@ -2,15 +2,16 @@ package app
 
 import (
 	"encoding/csv"
-	"os"
 	"strconv"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 
 	metadatatypes "github.com/provenance-io/provenance/x/metadata/types"
 )
 
-type NetAssetValueWithHeight struct {
+const umberTestnetScopeNAVsFN = "upgrade_files/umber/testnet_scope_navs.csv"
+
+type ScopeNAV struct {
 	ScopeUUID     string
 	NetAssetValue metadatatypes.NetAssetValue
 	Height        uint64
@@ -25,9 +26,9 @@ func parseValueToUsdMills(navStr string) (int64, error) {
 	return int64(navValue * 1000), nil
 }
 
-// ReadNetAssetValues reads a CSV file and parses its contents into a slice of NetAssetValueWithHeight
-func ReadNetAssetValues(fileName string) ([]NetAssetValueWithHeight, error) {
-	file, err := os.Open(fileName)
+// ReadScopeNAVs reads a CSV file from the upgrade_files dir, and parses its contents into a slice of ScopeNAV
+func ReadScopeNAVs(fileName string) ([]ScopeNAV, error) {
+	file, err := UpgradeFiles.Open(fileName)
 	if err != nil {
 		return nil, err
 	}
@@ -45,7 +46,7 @@ func ReadNetAssetValues(fileName string) ([]NetAssetValueWithHeight, error) {
 		return nil, err
 	}
 
-	assets := make([]NetAssetValueWithHeight, 0, len(records))
+	assets := make([]ScopeNAV, 0, len(records))
 	for _, record := range records {
 		if len(record) < 3 {
 			continue
@@ -66,7 +67,7 @@ func ReadNetAssetValues(fileName string) ([]NetAssetValueWithHeight, error) {
 
 		price := sdk.NewInt64Coin(metadatatypes.UsdDenom, navInt64)
 
-		asset := NetAssetValueWithHeight{
+		asset := ScopeNAV{
 			ScopeUUID:     scopeUUID,
 			NetAssetValue: metadatatypes.NewNetAssetValue(price),
 			Height:        height,

app/scope_navs_updater_test.go

@@ -3,35 +3,63 @@ package app
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	sdk "github.com/cosmos/cosmos-sdk/types"
+
 	metadatatypes "github.com/provenance-io/provenance/x/metadata/types"
-	"github.com/stretchr/testify/assert"
 )
 
-func TestReadNetAssetValues(t *testing.T) {
-	fileName := "upgrade_files/testnet_scope_navs.csv"
+func TestReadScopeNAVs(t *testing.T) {
+	tests := []struct {
+		fileName string
+		expCount int
+		expFirst ScopeNAV
+		expLast  ScopeNAV
+	}{
+		{
+			fileName: umberTestnetScopeNAVsFN,
+			expCount: 1101,
+			expFirst: ScopeNAV{
+				ScopeUUID:     "2f389a9f-873d-4920-85a6-7734f27e1738",
+				NetAssetValue: metadatatypes.NewNetAssetValue(sdk.NewInt64Coin(metadatatypes.UsdDenom, 398820670)),
+				Height:        23056719,
+			},
+			expLast: ScopeNAV{
+				ScopeUUID:     "65939db0-6d7a-42ef-9443-378304d33225",
+				NetAssetValue: metadatatypes.NewNetAssetValue(sdk.NewInt64Coin(metadatatypes.UsdDenom, 93661920)),
+				Height:        23056719,
+			},
+		},
+	}
+
+	assertEqualEntry := func(t *testing.T, expected, actual ScopeNAV, msg string, args ...interface{}) bool {
+		t.Helper()
+		if assert.Equalf(t, expected, actual, msg, args...) {
+			return true
+		}
 
-	assets, err := ReadNetAssetValues(fileName)
-	assert.NoError(t, err, "Failed to read net asset values")
-	assert.Len(t, assets, 1101, "The number of assets should be 1101")
+		assert.Equalf(t, expected.ScopeUUID, actual.ScopeUUID, msg+" ScopeUUID", args...)
+		if !assert.Equalf(t, expected.NetAssetValue, actual.NetAssetValue, msg+" NetAssetValue", args...) {
+			assert.Equalf(t, expected.NetAssetValue.Price.String(), actual.NetAssetValue.Price.String(), msg+" NetAssetValue.Price", args...)
+			assert.Equalf(t, int(expected.NetAssetValue.UpdatedBlockHeight), int(actual.NetAssetValue.UpdatedBlockHeight), msg+" NetAssetValue.UpdatedBlockHeight", args...)
+		}
+		assert.Equalf(t, expected.Height, actual.Height, msg+" Height", args...)
 
-	expectedFirst := NetAssetValueWithHeight{
-		ScopeUUID:     "2f389a9f-873d-4920-85a6-7734f27e1738",
-		NetAssetValue: metadatatypes.NewNetAssetValue(sdk.NewInt64Coin(metadatatypes.UsdDenom, 398820670)),
-		Height:        23056719,
+		return false
 	}
-	assert.Equal(t, expectedFirst.ScopeUUID, assets[0].ScopeUUID, "The first ScopeUUID should match")
-	assert.True(t, assets[0].NetAssetValue.Price.Equal(expectedFirst.NetAssetValue.Price), "The first NetAssetValue should match")
-	assert.Equal(t, expectedFirst.Height, assets[0].Height, "The first Height should match")
-
-	expectedLast := NetAssetValueWithHeight{
-		ScopeUUID:     "65939db0-6d7a-42ef-9443-378304d33225",
-		NetAssetValue: metadatatypes.NewNetAssetValue(sdk.NewInt64Coin(metadatatypes.UsdDenom, 93661920)),
-		Height:        23056719,
+
+	for _, tc := range tests {
+		t.Run(tc.fileName, func(t *testing.T) {
+			assets, err := ReadScopeNAVs(tc.fileName)
+			require.NoError(t, err, "Failed to read net asset values")
+			assert.Len(t, assets, tc.expCount, "The number of assets should be 1101")
+
+			assertEqualEntry(t, tc.expFirst, assets[0], "First entry")
+			assertEqualEntry(t, tc.expLast, assets[len(assets)-1], "Last entry")
+		})
 	}
-	assert.Equal(t, expectedLast.ScopeUUID, assets[len(assets)-1].ScopeUUID, "The last ScopeUUID should match")
-	assert.True(t, assets[len(assets)-1].NetAssetValue.Price.Equal(expectedLast.NetAssetValue.Price), "The last NetAssetValue should match")
-	assert.Equal(t, expectedLast.Height, assets[len(assets)-1].Height, "The last Height should match")
 }
 
 func TestParseValueToUsdMills(t *testing.T) {
@@ -44,17 +72,24 @@ func TestParseValueToUsdMills(t *testing.T) {
 		{"0.99", 990, false},
 		{"100.5", 100500, false},
 		{"100.3456", 100345, false},
+		{"172755", 172755000, false},
 		{"abc", 0, true},
 		{"", 0, true},
 	}
 
-	for _, test := range tests {
-		output, err := parseValueToUsdMills(test.input)
-		if test.expectError {
-			assert.Error(t, err, "Expected an error for input: %s", test.input)
-		} else {
-			assert.NoError(t, err, "Did not expect an error for input: %s", test.input)
-			assert.Equal(t, test.expectedOutput, output, "Expected output to be %d for input: %s", test.expectedOutput, test.input)
+	for _, tc := range tests {
+		name := tc.input
+		if len(name) == 0 {
+			name = "empty"
 		}
+		t.Run(name, func(t *testing.T) {
+			output, err := parseValueToUsdMills(tc.input)
+			if tc.expectError {
+				assert.Error(t, err, "parseValueToUsdMills error")
+			} else {
+				assert.NoError(t, err, "parseValueToUsdMills error")
+			}
+			assert.Equal(t, tc.expectedOutput, output, "parseValueToUsdMills output")
+		})
 	}
 }

app/upgrades.go

@@ -95,11 +95,10 @@ var upgrades = map[string]appUpgrade{
 
 			updateIBCClients(ctx, app)
 
-			scopeNavs, err := ReadNetAssetValues("upgrade_files/testnet_scope_navs.csv")
+			err = addScopeNAVs(ctx, app, umberTestnetScopeNAVsFN)
 			if err != nil {
 				return nil, err
 			}
-			addScopeNavsWithHeight(ctx, app, scopeNavs)
 
 			removeInactiveValidatorDelegations(ctx, app)
 
@@ -571,30 +570,48 @@ func setNewGovParams(ctx sdk.Context, app *App, newParams govv1.Params, chain st
 	return nil
 }
 
-// addScopeNavsWithHeight sets net asset values with heights for markers
+// addScopeNAVs reads a navs csv file and sets net asset values with heights for markers.
 // TODO: Remove with the umber handlers.
-func addScopeNavsWithHeight(ctx sdk.Context, app *App, navsWithHeight []NetAssetValueWithHeight) {
-	ctx.Logger().Info("Adding scope net asset values with heights.")
+func addScopeNAVs(ctx sdk.Context, app *App, fileName string) error {
+	ctx.Logger().Info("Adding scope net asset values.")
+	navs, err := ReadScopeNAVs(fileName)
+	if err != nil {
+		return fmt.Errorf("could not read navs: %w", err)
+	}
+	addScopeNAVsWithHeight(ctx, app, navs)
+	ctx.Logger().Info("Done adding scope net asset values.")
+	return nil
+}
+
+// addScopeNAVsWithHeight sets net asset values with heights for markers.
+// TODO: Remove with the umber handlers.
+func addScopeNAVsWithHeight(ctx sdk.Context, app *App, scopeNAVs []ScopeNAV) {
+	count := len(scopeNAVs)
+	ctx.Logger().Info(fmt.Sprintf("Adding %d scope net asset value entries.", count))
 
 	totalAdded := 0
-	for _, navWithHeight := range navsWithHeight {
-		uid, err := uuid.Parse(navWithHeight.ScopeUUID)
+	for i, scopeNAV := range scopeNAVs {
+		uid, err := uuid.Parse(scopeNAV.ScopeUUID)
 		if err != nil {
-			ctx.Logger().Error(fmt.Sprintf("invalid uuid %v : %v", navWithHeight.ScopeUUID, err))
+			ctx.Logger().Error(fmt.Sprintf("[%d/%d]: Invalid UUID %q: %v.", i+1, count, scopeNAV.ScopeUUID, err))
 			continue
 		}
 		scopeAddr := metadatatypes.ScopeMetadataAddress(uid)
 		_, found := app.MetadataKeeper.GetScope(ctx, scopeAddr)
 		if !found {
-			ctx.Logger().Error(fmt.Sprintf("unable to find scope %v", navWithHeight.ScopeUUID))
+			ctx.Logger().Error(fmt.Sprintf("[%d/%d]: Unable to find scope with UUID %q.", i+1, count, scopeNAV.ScopeUUID))
 			continue
 		}
 
-		if err := app.MetadataKeeper.SetNetAssetValueWithBlockHeight(ctx, scopeAddr, navWithHeight.NetAssetValue, "owner", navWithHeight.Height); err != nil {
-			ctx.Logger().Error(fmt.Sprintf("unable to set net asset value with height %v at height %d: %v", navWithHeight.NetAssetValue, navWithHeight.Height, err))
+		err = app.MetadataKeeper.SetNetAssetValueWithBlockHeight(ctx, scopeAddr, scopeNAV.NetAssetValue, "owner", scopeNAV.Height)
+		if err != nil {
+			ctx.Logger().Error(fmt.Sprintf("[%d/%d]: Unable to set scope %s (%q) net asset value %s at height %d: %v.",
+				i+1, count, scopeAddr, scopeNAV.ScopeUUID, scopeNAV.NetAssetValue.Price, scopeNAV.Height, err))
+			continue
 		}
+
 		totalAdded++
 	}
 
-	ctx.Logger().Info(fmt.Sprintf("Done adding a total of %v scope net asset values with heights.", totalAdded))
+	ctx.Logger().Info(fmt.Sprintf("Successfully added %d of %d scope net asset value entries.", totalAdded, count))
 }