From 193ed5881e3768ded26743e4fa89165d9bca974a Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sat, 3 Jun 2023 23:10:18 +0800
Subject: [PATCH 01/10] CNS-420: fixation to maintain DeleteAt on future
 entries too

Previously, entry.DeleteAt was tracked always on the latest entry, even if
there is a future entry with smaller block then this DeleteAt. Such case
prevented us from considering the this DeleteAt in lookup for future entries.

With this fix, entry.DeleteAt will now always be set on the entry with the
largets block that is still before the DeleteAt block (either the latest one
or a future one).
---
 common/fixation_entry.go | 56 +++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/common/fixation_entry.go b/common/fixation_entry.go
index d6b1999a45..fc3d6fc404 100644
--- a/common/fixation_entry.go
+++ b/common/fixation_entry.go
@@ -164,12 +164,12 @@ func (fs *FixationStore) getEntryStore(ctx sdk.Context, index string) *prefix.St
 	return &store
 }
 
-func (fs *FixationStore) replaceTimer(ctx sdk.Context, prev, next types.Entry, kind byte) {
+func (fs *FixationStore) replaceTimer(ctx sdk.Context, prev, next types.Entry, block uint64, kind byte) {
 	key := encodeForTimer(prev.Index, prev.Block, kind)
-	fs.tstore.DelTimerByBlockHeight(ctx, next.DeleteAt, key)
+	fs.tstore.DelTimerByBlockHeight(ctx, block, key)
 
 	key = encodeForTimer(next.Index, next.Block, kind)
-	fs.tstore.AddTimerByBlockHeight(ctx, next.DeleteAt, key, []byte{})
+	fs.tstore.AddTimerByBlockHeight(ctx, block, key, []byte{})
 }
 
 // getEntry returns an existing entry in the store
@@ -227,7 +227,7 @@ func (fs *FixationStore) AppendEntry(
 
 		// temporary: do not allow adding new entries for an index that was deleted
 		// and still not fully cleaned up (e.g. not stale or with references held)
-		if latestEntry.IsDeletedBy(ctxBlock) {
+		if latestEntry.IsDeletedBy(block) {
 			return utils.LavaFormatError("AppendEntry",
 				fmt.Errorf("entry already deleted and pending cleanup"),
 				utils.Attribute{Key: "index", Value: index},
@@ -241,17 +241,22 @@ func (fs *FixationStore) AppendEntry(
 			return nil
 		}
 
+		// if the previous latest entry is marked with DeleteAt which is set to expire after
+		// theis future entry's maturity (block), then transfer this DeleteAt to the future
+		// entry, and then replace the old timer with a new timer (below)
+		// (note: deletion, if any, cannot be for the current block, since it would have been
+		// processed at the beginning of the block, and AppendEntry would fail earlier).
+
+		if latestEntry.HasDeleteAt() { // already know !latestEntry.IsDeletedBy(block)
+			deleteAt = latestEntry.DeleteAt
+			latestEntry.DeleteAt = math.MaxUint64
+		}
+
 		// if we are superseding a previous latest entry, then drop the latter's refcount;
 		// otherwise we are a future entry version, so set a timer for when it will become
 		// the new latest entry.
-		// and if indeed we are, and the latest entry has DeleteAt set, then transfer this
-		// DeleteAt to the new entry, drop the old timer, and start a new timer. (note:
-		// deletion, if any, cannot be for the current block, because it would have been
-		// processed at the beginning of the block (and AppendEntry would fail earlier).
 
 		if block == ctxBlock {
-			deleteAt = latestEntry.DeleteAt
-			latestEntry.DeleteAt = math.MaxUint64
 			fs.putEntry(ctx, latestEntry)
 		} else {
 			key := encodeForTimer(safeIndex, block, timerFutureEntry)
@@ -273,7 +278,7 @@ func (fs *FixationStore) AppendEntry(
 	}
 
 	if entry.HasDeleteAt() {
-		fs.replaceTimer(ctx, latestEntry, entry, timerDeleteEntry)
+		fs.replaceTimer(ctx, latestEntry, entry, entry.DeleteAt, timerDeleteEntry)
 	}
 
 	fs.setEntry(ctx, entry)
@@ -302,28 +307,13 @@ func (fs *FixationStore) updateFutureEntry(ctx sdk.Context, safeIndex string, bl
 
 	latestEntry, found := fs.getUnmarshaledEntryForBlock(ctx, safeIndex, block-1)
 	if found {
-		// if the latest entry has DeleteAt set, then transfer this DeleteAt to this
-		// future entry (becoming latest). also, drop the old timer, and start a new
-		// timer. (note: // deletion, if any, cannot be for the current block, since
-		// it would have been processed at the beginning of the block.
+		// previous latest entry should never have its DeleteAt set for this block:
+		// if our AppendEntry happened before the DelEntry, we would get marked (and
+		// not the previous latest entry); if the DelEntry happened first, then we
+		// would inherit the DeleteAt from the previous latest entry.
 
 		if latestEntry.HasDeleteAt() {
-			entry := fs.getEntry(ctx, safeIndex, block)
-			entry.DeleteAt = latestEntry.DeleteAt
-			latestEntry.DeleteAt = math.MaxUint64
-			fs.setEntry(ctx, entry)
-
-			if entry.IsDeletedBy(block) {
-				// if DeleteAt is exactly now as we transition from future entry to
-				// be the latest entry, then simply invoke deleteMarkedEntry().
-				key := encodeForTimer(latestEntry.Index, latestEntry.Block, timerDeleteEntry)
-				fs.tstore.DelTimerByBlockHeight(ctx, block, key)
-				fs.deleteMarkedEntry(ctx, safeIndex, block)
-			} else {
-				// otherwise, then replace the old timer (for the previous latest)
-				// with a new timer (for the new latest).
-				fs.replaceTimer(ctx, latestEntry, entry, timerDeleteEntry)
-			}
+			panic(fmt.Sprintf("Future entry: latest entry has DeleteAt %d", latestEntry.DeleteAt))
 		}
 
 		// latest entry had extra refcount for being the latest; so drop that refcount
@@ -614,7 +604,7 @@ func (fs *FixationStore) DelEntry(ctx sdk.Context, index string, block uint64) e
 		panic(fmt.Sprintf("DelEntry for block %d < current ctx block %d", block, ctxBlock))
 	}
 
-	entry, found := fs.getUnmarshaledEntryForBlock(ctx, safeIndex, ctxBlock)
+	entry, found := fs.getUnmarshaledEntryForBlock(ctx, safeIndex, block)
 	if !found || entry.HasDeleteAt() {
 		return sdkerrors.ErrNotFound
 	}
@@ -626,7 +616,7 @@ func (fs *FixationStore) DelEntry(ctx sdk.Context, index string, block uint64) e
 	// to do the work. otherwise this is a future entry delete, so set a timer for
 	// when it will become imminent.
 
-	if block == uint64(ctx.BlockHeight()) {
+	if block == ctxBlock {
 		fs.deleteMarkedEntry(ctx, safeIndex, entry.Block)
 	} else {
 		key := encodeForTimer(safeIndex, entry.Block, timerDeleteEntry)

From 04965ffe6c909c025da663092fed26168c20af2f Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sat, 3 Jun 2023 23:24:56 +0800
Subject: [PATCH 02/10] CNS-420: refactor in x/projects/project_test.go

---
 x/projects/keeper/project_test.go | 377 ++++++++++++++++++------------
 1 file changed, 226 insertions(+), 151 deletions(-)

diff --git a/x/projects/keeper/project_test.go b/x/projects/keeper/project_test.go
index e412d7197f..c97f77c23e 100644
--- a/x/projects/keeper/project_test.go
+++ b/x/projects/keeper/project_test.go
@@ -3,6 +3,7 @@ package keeper_test
 import (
 	"context"
 	"math"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -16,27 +17,67 @@ import (
 
 const projectName = "mockname"
 
-func prepareProjectsData(ctx context.Context, keepers *testkeeper.Keepers) (projects []types.ProjectData) {
-	adm1Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	adm2Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	adm3Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	dev3Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
+type testStruct struct {
+	t        *testing.T
+	servers  *testkeeper.Servers
+	keepers  *testkeeper.Keepers
+	_ctx     context.Context
+	ctx      sdk.Context
+	accounts map[string]string
+	projects map[string]types.ProjectData
+}
+
+func newTestStruct(t *testing.T) *testStruct {
+	servers, keepers, _ctx := testkeeper.InitAllKeepers(t)
+	return &testStruct{
+		t:       t,
+		servers: servers,
+		keepers: keepers,
+		_ctx:    _ctx,
+		ctx:     sdk.UnwrapSDKContext(_ctx),
+	}
+}
+
+func (ts *testStruct) prepareData(numSub, numAdmin, numDevel int) {
+	ts.accounts = make(map[string]string)
+	for i := 0; i < numSub; i++ {
+		k := "sub" + strconv.Itoa(i+1)
+		v := common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
+		ts.accounts[k] = v
+	}
+	for i := 0; i < numAdmin; i++ {
+		k := "adm" + strconv.Itoa(i)
+		v := common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
+		ts.accounts[k] = v
+	}
+	for i := 0; i < numDevel; i++ {
+		k := "dev" + strconv.Itoa(i)
+		v := common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
+		ts.accounts[k] = v
+	}
+
+	ts.projects = make(map[string]types.ProjectData)
+
+	ts.accounts["pd1_adm"] = common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
+	ts.accounts["pd2_both"] = common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
+	ts.accounts["pd3_adm"] = common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
+	ts.accounts["pd3_dev"] = common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
 
 	// admin key
 	keys_1_admin := []types.ProjectKey{
-		types.ProjectAdminKey(adm1Addr),
+		types.ProjectAdminKey(ts.accounts["pd1_adm"]),
 	}
 	// developer key
 	keys_1_admin_dev := []types.ProjectKey{
-		types.NewProjectKey(adm2Addr).
+		types.NewProjectKey(ts.accounts["pd2_both"]).
 			AddType(types.ProjectKey_ADMIN).
 			AddType(types.ProjectKey_DEVELOPER),
 	}
 
 	// both (admin+developer) key
 	keys_2_admin_and_dev := []types.ProjectKey{
-		types.ProjectAdminKey(adm3Addr),
-		types.ProjectDeveloperKey(dev3Addr),
+		types.ProjectAdminKey(ts.accounts["pd3_adm"]),
+		types.ProjectDeveloperKey(ts.accounts["pd3_dev"]),
 	}
 
 	policy1 := &types.Policy{
@@ -45,69 +86,111 @@ func prepareProjectsData(ctx context.Context, keepers *testkeeper.Keepers) (proj
 	}
 
 	templates := []struct {
+		code    string
 		name    string
 		enabled bool
 		keys    []types.ProjectKey
 		policy  *types.Policy
 	}{
 		// project with admin key, enabled, has policy
-		{"mock_project_1", true, keys_1_admin, policy1},
+		{"pd1", "mock_project_1", true, keys_1_admin, policy1},
 		// project with "both" key, disabled, with policy
-		{"mock_project_2", false, keys_1_admin_dev, policy1},
+		{"pd2", "mock_project_2", false, keys_1_admin_dev, policy1},
 		// project with 2 keys (one admin, one developer) disabled, no policy
-		{"mock_project_3", false, keys_2_admin_and_dev, nil},
+		{"pd3", "mock_project_3", false, keys_2_admin_and_dev, nil},
 	}
 
 	for _, tt := range templates {
-		projectData := types.ProjectData{
+		ts.projects[tt.code] = types.ProjectData{
 			Name:        tt.name,
 			Description: "",
 			Enabled:     tt.enabled,
 			ProjectKeys: tt.keys,
 			Policy:      tt.policy,
 		}
-		projects = append(projects, projectData)
 	}
+}
+
+func (ts *testStruct) BlockHeight() uint64 {
+	return uint64(ts.ctx.BlockHeight())
+}
+
+func (ts *testStruct) AdvanceBlock(count int) {
+	for i := 0; i < count; i += 1 {
+		ts._ctx = testkeeper.AdvanceBlock(ts._ctx, ts.keepers)
+	}
+	ts.ctx = sdk.UnwrapSDKContext(ts._ctx)
+}
+
+func (ts *testStruct) AdvanceEpoch(count int) {
+	for i := 0; i < count; i += 1 {
+		ts._ctx = testkeeper.AdvanceEpoch(ts._ctx, ts.keepers)
+	}
+	ts.ctx = sdk.UnwrapSDKContext(ts._ctx)
+}
+
+func (ts *testStruct) isKeyInProject(index, key string, kind types.ProjectKey_Type) bool {
+	resp, err := ts.keepers.Projects.Info(ts._ctx, &types.QueryInfoRequest{Project: index})
+	require.Nil(ts.t, err, "project: " + index + ", key: " + key)
+	pk := resp.Project.GetKey(key)
+	return pk.IsType(kind)
+}
+
+func (ts *testStruct) addProjectKeys(index, creator string, projectKeys ...types.ProjectKey) error {
+	msg := types.MsgAddKeys{
+		Creator: creator,
+		Project: index,
+		ProjectKeys: projectKeys,
+	}
+	_, err := ts.servers.ProjectServer.AddKeys(ts._ctx, &msg)
+	return err
+}
 
-	return projects
+func (ts *testStruct) delProjectKeys(index, creator string, projectKeys ...types.ProjectKey) error {
+	msg := types.MsgDelKeys{
+		Creator: creator,
+		Project: index,
+		ProjectKeys: projectKeys,
+	}
+	_, err := ts.servers.ProjectServer.DelKeys(ts._ctx, &msg)
+	return err
 }
 
 func TestCreateDefaultProject(t *testing.T) {
-	_, keepers, ctx := testkeeper.InitAllKeepers(t)
+	ts := newTestStruct(t)
 
-	subAddr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
+	subAddr := common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
 	plan := common.CreateMockPlan()
 
-	err := keepers.Projects.CreateAdminProject(sdk.UnwrapSDKContext(ctx), subAddr, plan)
+	err := ts.keepers.Projects.CreateAdminProject(ts.ctx, subAddr, plan)
 	require.Nil(t, err)
 
 	// subscription key is a developer in the default project
-	response1, err := keepers.Projects.Developer(ctx, &types.QueryDeveloperRequest{Developer: subAddr})
+	response1, err := ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: subAddr})
 	require.Nil(t, err)
 
-	ctx = testkeeper.AdvanceEpoch(ctx, keepers)
+	ts.AdvanceEpoch(1)
 
-	response2, err := keepers.Projects.Info(ctx, &types.QueryInfoRequest{Project: response1.Project.Index})
+	response2, err := ts.keepers.Projects.Info(ts._ctx, &types.QueryInfoRequest{Project: response1.Project.Index})
 	require.Nil(t, err)
 
 	require.Equal(t, response2.Project, response1.Project)
 }
 
 func TestCreateProject(t *testing.T) {
-	servers, keepers, _ctx := testkeeper.InitAllKeepers(t)
-	ctx := sdk.UnwrapSDKContext(_ctx)
+	ts := newTestStruct(t)
+	ts.prepareData(1, 0, 0) // 1 sub, 0 adm, 0 dev
 
-	projectData := prepareProjectsData(_ctx, keepers)[1]
+	projectData := ts.projects["pd1"]
 	plan := common.CreateMockPlan()
 
-	subAddr := common.CreateNewAccount(_ctx, *keepers, 10000).Addr.String()
-	admAddr := projectData.ProjectKeys[0].Key
+	subAddr := ts.accounts["sub1"]
+	admAddr := ts.accounts["pd2_adm"]
 
-	err := keepers.Projects.CreateProject(ctx, subAddr, projectData, plan)
+	err := ts.keepers.Projects.CreateProject(ts.ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
 
-	_ctx = testkeeper.AdvanceEpoch(_ctx, keepers)
-	ctx = sdk.UnwrapSDKContext(_ctx)
+	ts.AdvanceEpoch(1)
 
 	// test invalid project name/description
 	defaultProjectName := types.ADMIN_PROJECT_NAME
@@ -139,7 +222,7 @@ func TestCreateProject(t *testing.T) {
 			testProjectData.Name = tt.projectName
 			testProjectData.Description = tt.projectDescription
 
-			err = keepers.Projects.CreateProject(ctx, subAddr, testProjectData, plan)
+			err = ts.keepers.Projects.CreateProject(ts.ctx, subAddr, testProjectData, plan)
 			require.NotNil(t, err)
 		})
 	}
@@ -154,26 +237,26 @@ func TestCreateProject(t *testing.T) {
 	}
 
 	// should fail because there's an invalid key
-	_, err = servers.SubscriptionServer.AddProject(_ctx, &subscriptiontypes.MsgAddProject{
+	_, err = ts.servers.SubscriptionServer.AddProject(ts._ctx, &subscriptiontypes.MsgAddProject{
 		Creator:     subAddr,
 		ProjectData: invalidKeysProjectData,
 	})
 	require.NotNil(t, err)
 
-	// get project by developer - subscription key is not a developer, should fail (if it succeeds, it means that the valid project key
-	// from invalidKeysProjectData was registered, which is not desired!)
-	_, err = keepers.Projects.Developer(_ctx, &types.QueryDeveloperRequest{Developer: subAddr})
+	// subscription key is not a developer, so get project by developer should fail (if it succeeds,
+	// then the valid project key from invalidKeysProjectData was registered, which is undesired!)
+	_, err = ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: subAddr})
 	require.NotNil(t, err)
 
-	response1, err := keepers.Projects.Developer(_ctx, &types.QueryDeveloperRequest{Developer: admAddr})
+	response1, err := ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: admAddr})
 	require.Nil(t, err)
 
-	response2, err := keepers.Projects.Info(_ctx, &types.QueryInfoRequest{Project: response1.Project.Index})
+	response2, err := ts.keepers.Projects.Info(ts._ctx, &types.QueryInfoRequest{Project: response1.Project.Index})
 	require.Nil(t, err)
 
 	require.Equal(t, response2.Project, response1.Project)
 
-	_, err = keepers.Projects.GetProjectForBlock(ctx, response1.Project.Index, 0)
+	_, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, response1.Project.Index, 0)
 	require.Nil(t, err)
 
 	// there should be one project key
@@ -188,85 +271,87 @@ func TestCreateProject(t *testing.T) {
 }
 
 func TestAddKeys(t *testing.T) {
-	servers, keepers, ctx := testkeeper.InitAllKeepers(t)
-	_ctx := sdk.UnwrapSDKContext(ctx)
-	projectData := prepareProjectsData(ctx, keepers)[2]
+	ts := newTestStruct(t)
+	ts.prepareData(1, 0, 2) // 1 sub, 0 adm, 2 dev
+
+	projectData := ts.projects["pd2"]
 	plan := common.CreateMockPlan()
 
-	subAddr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	admAddr := projectData.ProjectKeys[0].Key
-	dev1Addr := projectData.ProjectKeys[1].Key
-	dev2Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	dev3Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
+	subAddr := ts.accounts["sub1"]
+	admAddr := ts.accounts["pd3_adm"]
+	dev1Addr := ts.accounts["pd3_dev"]
+	dev2Addr := ts.accounts["dev1"]
+	dev3Addr := ts.accounts["dev2"]
 
-	err := keepers.Projects.CreateProject(_ctx, subAddr, projectData, plan)
+	err := ts.keepers.Projects.CreateProject(ts.ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
 
-	ctx = testkeeper.AdvanceEpoch(ctx, keepers)
+	ts.AdvanceEpoch(1)
 
-	projectRes, err := keepers.Projects.Developer(ctx, &types.QueryDeveloperRequest{Developer: dev1Addr})
+	projectRes, err := ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: dev1Addr})
 	require.Nil(t, err)
 
 	project := projectRes.Project
 	pk := types.ProjectAdminKey(dev1Addr)
 
 	// try adding myself as admin, should fail
-	_, err = servers.ProjectServer.AddKeys(ctx, &types.MsgAddKeys{Creator: dev1Addr, Project: project.Index, ProjectKeys: []types.ProjectKey{pk}})
+	err = ts.addProjectKeys(project.Index, dev1Addr, pk)
 	require.NotNil(t, err)
 
 	// admin key adding an invalid key
 	pk = types.NewProjectKey(dev2Addr).AddType(0x4)
-	_, err = servers.ProjectServer.AddKeys(ctx, &types.MsgAddKeys{Creator: admAddr, Project: project.Index, ProjectKeys: []types.ProjectKey{pk}})
+	err = ts.addProjectKeys(project.Index, admAddr, pk)
 	require.NotNil(t, err)
 
 	// admin key adding a developer
 	pk = types.ProjectDeveloperKey(dev2Addr)
-	_, err = servers.ProjectServer.AddKeys(ctx, &types.MsgAddKeys{Creator: admAddr, Project: project.Index, ProjectKeys: []types.ProjectKey{pk}})
+	err = ts.addProjectKeys(project.Index, admAddr, pk)
 	require.Nil(t, err)
 
 	// developer tries to add the second developer as admin
 	pk = types.ProjectAdminKey(dev2Addr)
-	_, err = servers.ProjectServer.AddKeys(ctx, &types.MsgAddKeys{Creator: dev1Addr, Project: project.Index, ProjectKeys: []types.ProjectKey{pk}})
+	err = ts.addProjectKeys(project.Index, dev1Addr, pk)
 	require.NotNil(t, err)
 
 	// admin adding admin
 	pk = types.ProjectAdminKey(dev1Addr)
-	_, err = servers.ProjectServer.AddKeys(ctx, &types.MsgAddKeys{Creator: admAddr, Project: project.Index, ProjectKeys: []types.ProjectKey{pk}})
+	err = ts.addProjectKeys(project.Index, admAddr, pk)
 	require.Nil(t, err)
 
 	// new admin adding another developer
 	pk = types.ProjectDeveloperKey(dev3Addr)
-	_, err = servers.ProjectServer.AddKeys(ctx, &types.MsgAddKeys{Creator: dev1Addr, Project: project.Index, ProjectKeys: []types.ProjectKey{pk}})
+	err = ts.addProjectKeys(project.Index, dev1Addr, pk)
 	require.Nil(t, err)
 
 	// fetch project with new developer
-	_, err = keepers.Projects.Developer(ctx, &types.QueryDeveloperRequest{Developer: dev3Addr})
+	_, err = ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: dev3Addr})
 	require.Nil(t, err)
 }
 
 func TestAddAdminInTwoProjects(t *testing.T) {
-	_, keepers, ctx := testkeeper.InitAllKeepers(t)
-	_ctx := sdk.UnwrapSDKContext(ctx)
-	projectData := prepareProjectsData(ctx, keepers)[0]
+	ts := newTestStruct(t)
+	ts.prepareData(1, 0, 0) // 1 sub, 0 admin, 0 devel
+
+	projectData := ts.projects["pd1"]
 	plan := common.CreateMockPlan()
 
-	subAddr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	admAddr := projectData.ProjectKeys[0].Key
+	subAddr := ts.accounts["sub1"]
+	admAddr := ts.accounts["pd1_adm"]
 
-	err := keepers.Projects.CreateAdminProject(_ctx, subAddr, plan)
+	err := ts.keepers.Projects.CreateAdminProject(ts.ctx, subAddr, plan)
 	require.Nil(t, err)
 
 	// this is not supposed to fail because you can use the same admin key for two different projects
 	// creating a regular project (not admin project) so subAccount won't be a developer there
-	err = keepers.Projects.CreateProject(_ctx, subAddr, projectData, plan)
+	err = ts.keepers.Projects.CreateProject(ts.ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
 
-	ctx = testkeeper.AdvanceEpoch(ctx, keepers)
+	ts.AdvanceEpoch(1)
 
-	_, err = keepers.Projects.Developer(ctx, &types.QueryDeveloperRequest{Developer: admAddr})
+	_, err = ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: admAddr})
 	require.NotNil(t, err)
 
-	response, err := keepers.Projects.Developer(ctx, &types.QueryDeveloperRequest{Developer: subAddr})
+	response, err := ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: subAddr})
 	require.Nil(t, err)
 	require.Equal(t, response.Project.Index, types.ProjectIndex(subAddr, types.ADMIN_PROJECT_NAME))
 }
@@ -280,27 +365,27 @@ func TestSetSubscriptionPolicy(t *testing.T) {
 }
 
 func SetPolicyTest(t *testing.T, testAdminPolicy bool) {
-	servers, keepers, _ctx := testkeeper.InitAllKeepers(t)
-	ctx := sdk.UnwrapSDKContext(_ctx)
+	ts := newTestStruct(t)
+	ts.prepareData(1, 0, 1) // 1 sub, 0 admin, 1 data
 
-	projectData := prepareProjectsData(_ctx, keepers)[0]
+	projectData := ts.projects["pd1"]
 	plan := common.CreateMockPlan()
 
-	subAddr := common.CreateNewAccount(_ctx, *keepers, 10000).Addr.String()
-	admAddr := projectData.ProjectKeys[0].Key
-	devAddr := common.CreateNewAccount(_ctx, *keepers, 10000).Addr.String()
+	subAddr := ts.accounts["sub1"]
+	admAddr := ts.accounts["pd1_adm"]
+	devAddr := ts.accounts["dev1"]
 
 	projectID := types.ProjectIndex(subAddr, projectData.Name)
 
-	err := keepers.Projects.CreateProject(ctx, subAddr, projectData, plan)
+	err := ts.keepers.Projects.CreateProject(ts.ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
 
 	pk := types.ProjectDeveloperKey(devAddr)
-	err = keepers.Projects.AddKeysToProject(ctx, projectID, admAddr, []types.ProjectKey{pk})
+	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID, admAddr, []types.ProjectKey{pk})
 	require.Nil(t, err)
 
 	spec := common.CreateMockSpec()
-	keepers.Spec.SetSpec(ctx, spec)
+	ts.keepers.Spec.SetSpec(ts.ctx, spec)
 
 	templates := []struct {
 		name                         string
@@ -395,13 +480,12 @@ func SetPolicyTest(t *testing.T, testAdminPolicy bool) {
 				err = SetPolicyMessage.ValidateBasic()
 				require.Nil(t, err)
 
-				_, err := servers.ProjectServer.SetPolicy(_ctx, &SetPolicyMessage)
+				_, err := ts.servers.ProjectServer.SetPolicy(ts._ctx, &SetPolicyMessage)
 				if tt.setAdminPolicySuccess {
 					require.Nil(t, err)
-					_ctx = testkeeper.AdvanceEpoch(_ctx, keepers)
-					ctx = sdk.UnwrapSDKContext(_ctx)
+					ts.AdvanceEpoch(1)
 
-					proj, err := keepers.Projects.GetProjectForBlock(ctx, tt.projectID, uint64(ctx.BlockHeight()))
+					proj, err := ts.keepers.Projects.GetProjectForBlock(ts.ctx, tt.projectID, ts.BlockHeight())
 					require.Nil(t, err)
 
 					require.Equal(t, newPolicy, *proj.AdminPolicy)
@@ -418,15 +502,13 @@ func SetPolicyTest(t *testing.T, testAdminPolicy bool) {
 				err = setSubscriptionPolicyMessage.ValidateBasic()
 				require.Nil(t, err)
 
-				_, err := servers.ProjectServer.SetSubscriptionPolicy(_ctx, &setSubscriptionPolicyMessage)
+				_, err := ts.servers.ProjectServer.SetSubscriptionPolicy(ts._ctx, &setSubscriptionPolicyMessage)
 				if tt.setSubscriptionPolicySuccess {
 					require.Nil(t, err)
-					_ctx = testkeeper.AdvanceEpoch(_ctx, keepers)
-					ctx = sdk.UnwrapSDKContext(_ctx)
+					ts.AdvanceEpoch(1)
 
-					proj, err := keepers.Projects.GetProjectForBlock(ctx, tt.projectID, uint64(ctx.BlockHeight()))
+					proj, err := ts.keepers.Projects.GetProjectForBlock(ts.ctx, tt.projectID, ts.BlockHeight())
 					require.Nil(t, err)
-
 					require.Equal(t, newPolicy, *proj.SubscriptionPolicy)
 				} else {
 					require.NotNil(t, err)
@@ -437,85 +519,79 @@ func SetPolicyTest(t *testing.T, testAdminPolicy bool) {
 }
 
 func TestChargeComputeUnits(t *testing.T) {
-	servers, keepers, _ctx := testkeeper.InitAllKeepers(t)
+	ts := newTestStruct(t)
+	ts.prepareData(0, 0, 1) // 0 sub, 0 adm, 1 dev
 
-	projectData := prepareProjectsData(_ctx, keepers)[0]
+	projectData := ts.projects["pd1"]
 	plan := common.CreateMockPlan()
 
-	subAddr := projectData.ProjectKeys[0].Key
-	devAddr := common.CreateNewAccount(_ctx, *keepers, 10000).Addr.String()
+	subAddr := ts.accounts["pd1_adm"]
+	devAddr := ts.accounts["dev1"]
 
-	_ctx = testkeeper.AdvanceEpoch(_ctx, keepers)
-	ctx := sdk.UnwrapSDKContext(_ctx)
-	block1 := uint64(ctx.BlockHeight())
+	ts.AdvanceEpoch(1)
+	block1 := ts.BlockHeight()
 
-	err := keepers.Projects.CreateProject(ctx, subAddr, projectData, plan)
+	err := ts.keepers.Projects.CreateProject(ts.ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
 
-	_ctx = testkeeper.AdvanceEpoch(_ctx, keepers)
-	ctx = sdk.UnwrapSDKContext(_ctx)
-	block2 := uint64(ctx.BlockHeight())
+	ts.AdvanceEpoch(1)
+	block2 := ts.BlockHeight()
 
 	projectID := types.ProjectIndex(subAddr, projectData.Name)
-	project, err := keepers.Projects.GetProjectForBlock(ctx, projectID, block2)
+	project, err := ts.keepers.Projects.GetProjectForBlock(ts.ctx, projectID, block2)
 	require.Nil(t, err)
 
 	// add developer key (created fixation)
-	pk := types.ProjectDeveloperKey(devAddr)
-	_, err = servers.ProjectServer.AddKeys(_ctx, &types.MsgAddKeys{
-		Creator:     subAddr,
-		Project:     project.Index,
-		ProjectKeys: []types.ProjectKey{pk},
-	})
+	err = ts.addProjectKeys(project.Index, subAddr, types.ProjectDeveloperKey(devAddr))
 	require.Nil(t, err)
 
-	_ctx = testkeeper.AdvanceEpoch(_ctx, keepers)
-	ctx = sdk.UnwrapSDKContext(_ctx)
-	block3 := uint64(ctx.BlockHeight())
 
-	keepers.Projects.SnapshotSubscriptionProjects(ctx, subAddr)
+	ts.AdvanceEpoch(1)
+	block3 := ts.BlockHeight()
+
+	ts.keepers.Projects.SnapshotSubscriptionProjects(ts.ctx, subAddr)
 
 	// try to charge CUs: should update oldest and second-oldest entries, but not the latest
 	// (because the latter is in a new snapshot)
 
-	err = keepers.Projects.ChargeComputeUnitsToProject(ctx, project, block1, 1000)
+	err = ts.keepers.Projects.ChargeComputeUnitsToProject(ts.ctx, project, block1, 1000)
 	require.Nil(t, err)
 
-	proj, err := keepers.Projects.GetProjectForBlock(ctx, project.Index, block1)
+	proj, err := ts.keepers.Projects.GetProjectForBlock(ts.ctx, project.Index, block1)
 	require.Nil(t, err)
 	require.Equal(t, uint64(1000), proj.UsedCu)
-	proj, err = keepers.Projects.GetProjectForBlock(ctx, project.Index, block2)
+	proj, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, project.Index, block2)
 	require.Nil(t, err)
 	require.Equal(t, uint64(1000), proj.UsedCu)
-	proj, err = keepers.Projects.GetProjectForBlock(ctx, project.Index, block3)
+	proj, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, project.Index, block3)
 	require.Nil(t, err)
 	require.Equal(t, uint64(0), proj.UsedCu)
 
-	keepers.Projects.ChargeComputeUnitsToProject(ctx, project, block2, 1000)
+	ts.keepers.Projects.ChargeComputeUnitsToProject(ts.ctx, project, block2, 1000)
 
-	proj, err = keepers.Projects.GetProjectForBlock(ctx, project.Index, block1)
+	proj, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, project.Index, block1)
 	require.Nil(t, err)
 	require.Equal(t, uint64(1000), proj.UsedCu)
-	proj, err = keepers.Projects.GetProjectForBlock(ctx, project.Index, block2)
+	proj, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, project.Index, block2)
 	require.Nil(t, err)
 	require.Equal(t, uint64(2000), proj.UsedCu)
-	proj, err = keepers.Projects.GetProjectForBlock(ctx, project.Index, block3)
+	proj, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, project.Index, block3)
 	require.Nil(t, err)
 	require.Equal(t, uint64(0), proj.UsedCu)
 }
 
-func TestAddDevKeyToSameProjectDifferentBlocks(t *testing.T) {
-	_, keepers, ctx := testkeeper.InitAllKeepers(t)
-	_ctx := sdk.UnwrapSDKContext(ctx)
-	projectName := "mockname1"
-	subAddr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	dev1Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	dev2Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	projectID := types.ProjectIndex(subAddr, projectName)
+func TestAddDevKeysToSameProjectID(t *testing.T) {
+	ts := newTestStruct(t)
+	ts.prepareData(2, 1, 5) // 2 sub, 1 adm, 5 dev
+
+	sub Addr := ts.accounts["sub1"]
+	dev1Addr := ts.accounts["dev1"]
+	dev2Addr := ts.accounts["dev2"]
+
 	plan := common.CreateMockPlan()
 
-	projectData := types.ProjectData{
-		Name:        projectName,
+	projectData1 := types.ProjectData{
+		Name:        "mockname1",
 		Description: "",
 		Enabled:     true,
 		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(subAddr)},
@@ -524,35 +600,36 @@ func TestAddDevKeyToSameProjectDifferentBlocks(t *testing.T) {
 	err := keepers.Projects.CreateProject(_ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
 
-	ctx = testkeeper.AdvanceBlock(ctx, keepers)
-	_ctx = sdk.UnwrapSDKContext(ctx)
 
-	err = keepers.Projects.AddKeysToProject(_ctx, projectID, subAddr,
+	projectID := types.ProjectIndex(sub1Addr, projectData1.Name)
+
+	ts.AdvanceBlock(1)
+
+	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID, sub1Addr,
 		[]types.ProjectKey{types.ProjectDeveloperKey(dev1Addr)})
 	require.Nil(t, err)
 
-	ctx = testkeeper.AdvanceBlock(ctx, keepers)
-	_ctx = sdk.UnwrapSDKContext(ctx)
+	ts.AdvanceBlock(1)
 
-	err = keepers.Projects.AddKeysToProject(_ctx, projectID, subAddr,
+	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID, sub1Addr,
 		[]types.ProjectKey{types.ProjectDeveloperKey(dev2Addr)})
 	require.Nil(t, err)
 
-	proj, err := keepers.Projects.GetProjectForDeveloper(_ctx, subAddr,
-		uint64(_ctx.BlockHeight()))
-	require.Nil(t, err)
 
+	proj, err := ts.keepers.Projects.GetProjectForDeveloper(ts.ctx, subAddr, ts.BlockHeight())
+	require.Nil(t, err)
 	require.Equal(t, 3, len(proj.ProjectKeys))
 }
 
 func TestAddDevKeyToDifferentProjectsInSameBlock(t *testing.T) {
-	_, keepers, ctx := testkeeper.InitAllKeepers(t)
-	_ctx := sdk.UnwrapSDKContext(ctx)
-	plan := common.CreateMockPlan()
+	ts := newTestStruct(t)
+	ts.prepareData(2, 0, 1) // 2 sub, 0 adm, 1 dev
+
+	sub1Addr := ts.accounts["sub1"]
+	sub2Addr := ts.accounts["sub2"]
+	dev1Addr := ts.accounts["dev1"]
 
-	sub1Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	sub2Addr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
-	devAddr := common.CreateNewAccount(ctx, *keepers, 10000).Addr.String()
+	plan := common.CreateMockPlan()
 
 	projectName1 := "mockname1"
 	projectName2 := "mockname2"
@@ -567,7 +644,7 @@ func TestAddDevKeyToDifferentProjectsInSameBlock(t *testing.T) {
 		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(sub1Addr)},
 		Policy:      &plan.PlanPolicy,
 	}
-	err := keepers.Projects.CreateProject(_ctx, sub1Addr, projectData1, plan)
+	err := ts.keepers.Projects.CreateProject(ts.ctx, sub1Addr, projectData1, plan)
 	require.Nil(t, err)
 
 	projectData2 := types.ProjectData{
@@ -577,26 +654,24 @@ func TestAddDevKeyToDifferentProjectsInSameBlock(t *testing.T) {
 		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(sub2Addr)},
 		Policy:      &plan.PlanPolicy,
 	}
-	err = keepers.Projects.CreateProject(_ctx, sub2Addr, projectData2, plan)
+	err = ts.keepers.Projects.CreateProject(ts.ctx, sub2Addr, projectData2, plan)
 	require.Nil(t, err)
 
-	ctx = testkeeper.AdvanceBlock(ctx, keepers)
-	_ctx = sdk.UnwrapSDKContext(ctx)
+	ts.AdvanceBlock(1)
 
-	err = keepers.Projects.AddKeysToProject(_ctx, projectID1, sub1Addr,
-		[]types.ProjectKey{types.ProjectDeveloperKey(devAddr)})
+	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID1, sub1Addr,
+		[]types.ProjectKey{types.ProjectDeveloperKey(dev1Addr)})
 	require.Nil(t, err)
 
-	err = keepers.Projects.AddKeysToProject(_ctx, projectID2, sub2Addr,
-		[]types.ProjectKey{types.ProjectDeveloperKey(devAddr)})
-	require.NotNil(t, err) // should fail since this developer was already added to the first project
+	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID2, sub2Addr,
+		[]types.ProjectKey{types.ProjectDeveloperKey(dev1Addr)})
+	require.NotNil(t, err) // developer was already added to the first project
+
 
-	proj1, err := keepers.Projects.GetProjectForDeveloper(_ctx, sub1Addr,
-		uint64(_ctx.BlockHeight()))
+	proj1, err := ts.keepers.Projects.GetProjectForDeveloper(ts.ctx, sub1Addr, ts.BlockHeight())
 	require.Nil(t, err)
 
-	proj2, err := keepers.Projects.GetProjectForDeveloper(_ctx, sub2Addr,
-		uint64(_ctx.BlockHeight()))
+	proj2, err := ts.keepers.Projects.GetProjectForDeveloper(ts.ctx, sub2Addr, ts.BlockHeight())
 	require.Nil(t, err)
 
 	require.Equal(t, 2, len(proj1.ProjectKeys))

From 5db6bb891b2c836676e4ae73f0f6933317f80c68 Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sat, 3 Jun 2023 23:30:58 +0800
Subject: [PATCH 03/10] CNS-420: support DelKeysFromProjects() core logic

---
 x/projects/keeper/creation.go | 104 +++++++++++++++++++++++--------
 x/projects/keeper/keeper.go   |   6 +-
 x/projects/keeper/project.go  | 112 +++++++++++++++++++++++++++++-----
 x/projects/types/project.go   |  22 ++++++-
 4 files changed, 200 insertions(+), 44 deletions(-)

diff --git a/x/projects/keeper/creation.go b/x/projects/keeper/creation.go
index 3df8443d49..ff9f6a15f6 100644
--- a/x/projects/keeper/creation.go
+++ b/x/projects/keeper/creation.go
@@ -30,8 +30,8 @@ func (k Keeper) CreateProject(ctx sdk.Context, subAddr string, projectData types
 	}
 
 	var emptyProject types.Project
-	blockHeight := uint64(ctx.BlockHeight())
-	if found := k.projectsFS.FindEntry(ctx, project.Index, blockHeight, &emptyProject); found {
+	ctxBlock := uint64(ctx.BlockHeight())
+	if found := k.projectsFS.FindEntry(ctx, project.Index, ctxBlock, &emptyProject); found {
 		// the project with the same name already exists if no error has returned
 		return utils.LavaFormatWarning(
 			"project already exist for the current subscription with the same name",
@@ -47,13 +47,13 @@ func (k Keeper) CreateProject(ctx sdk.Context, subAddr string, projectData types
 	project.SubscriptionPolicy = project.AdminPolicy
 
 	for _, projectKey := range projectData.GetProjectKeys() {
-		err = k.registerKey(ctx, projectKey, &project, blockHeight)
+		err = k.registerKey(ctx, projectKey, &project, ctxBlock)
 		if err != nil {
 			return err
 		}
 	}
 
-	return k.projectsFS.AppendEntry(ctx, project.Index, blockHeight, &project)
+	return k.projectsFS.AppendEntry(ctx, project.Index, ctxBlock, &project)
 }
 
 func (k Keeper) registerKey(ctx sdk.Context, key types.ProjectKey, project *types.Project, blockHeight uint64) error {
@@ -62,52 +62,106 @@ func (k Keeper) registerKey(ctx sdk.Context, key types.ProjectKey, project *type
 	}
 
 	if key.IsType(types.ProjectKey_ADMIN) {
-		k.addAdminKey(project, key.GetKey())
+		project.AppendKey(types.ProjectAdminKey(key.Key))
 	}
 
 	if key.IsType(types.ProjectKey_DEVELOPER) {
+		ctxBlock := uint64(ctx.BlockHeight())
+
+		// check that the developer key is valid in the current project state
 		var devkeyData types.ProtoDeveloperData
-		found := k.developerKeysFS.FindEntry(ctx, key.GetKey(), blockHeight, &devkeyData)
+		found := k.developerKeysFS.FindEntry(ctx, key.Key, ctxBlock, &devkeyData)
+
+		// the developer key may already belong to a different project
+		if found && devkeyData.ProjectID != project.GetIndex() {
+			return utils.LavaFormatWarning("failed to register key",
+				fmt.Errorf("key already exists"),
+				utils.Attribute{Key: "key", Value: key.Key},
+				utils.Attribute{Key: "keyTypes", Value: key.Kinds},
+			)
+		}
+
+		// the project may have future (e.g. end of epoch) changes pending; so
+		// check that the developer key is still valid in that future state
+		// (for example, it could be removed and added elsewhere by then).
+		found = k.developerKeysFS.FindEntry(ctx, key.Key, blockHeight, &devkeyData)
 
-		// the developer key may not already belong to a different project
+		// the developer key may already belong to a different project
+		devkeyData = types.ProtoDeveloperData{}
 		if found && devkeyData.ProjectID != project.GetIndex() {
-			return utils.LavaFormatWarning("key already exists",
-				fmt.Errorf("could not register key to project"),
+			return utils.LavaFormatWarning("failed to register key",
+				fmt.Errorf("key already exists in next epoch"),
 				utils.Attribute{Key: "key", Value: key.Key},
 				utils.Attribute{Key: "keyTypes", Value: key.Kinds},
 			)
 		}
 
 		if !found {
-			err := k.addDeveloperKey(ctx, key.GetKey(), project, blockHeight)
+			devkeyData := types.ProtoDeveloperData{
+				ProjectID: project.GetIndex(),
+			}
+
+			err := k.developerKeysFS.AppendEntry(ctx, key.Key, blockHeight, &devkeyData)
 			if err != nil {
-				return utils.LavaFormatError("adding developer key to project failed", err,
-					utils.Attribute{Key: "developerKey", Value: key.Key},
-					utils.Attribute{Key: "projectIndex", Value: project.Index},
-					utils.Attribute{Key: "blockHeight", Value: blockHeight},
+				return utils.LavaFormatError("failed to register key", err,
+					utils.Attribute{Key: "key", Value: key.Key},
+					utils.Attribute{Key: "keyTypes", Value: key.Kinds},
 				)
 			}
+
+			project.AppendKey(types.ProjectDeveloperKey(key.Key))
 		}
 	}
 
 	return nil
 }
 
-func (k Keeper) addAdminKey(project *types.Project, adminKey string) {
-	project.AppendKey(types.ProjectAdminKey(adminKey))
-}
-
-func (k Keeper) addDeveloperKey(ctx sdk.Context, devkey string, project *types.Project, blockHeight uint64) error {
-	devkeyData := types.ProtoDeveloperData{
-		ProjectID: project.GetIndex(),
+func (k Keeper) unregisterKey(ctx sdk.Context, key types.ProjectKey, project *types.Project, blockHeight uint64) error {
+	if !key.IsTypeValid() {
+		return sdkerrors.ErrInvalidType
 	}
 
-	err := k.developerKeysFS.AppendEntry(ctx, devkey, blockHeight, &devkeyData)
-	if err != nil {
-		return err
+	if key.IsType(types.ProjectKey_ADMIN) {
+		found := project.DeleteKey(types.ProjectAdminKey(key.Key))
+		if !found {
+			return sdkerrors.ErrKeyNotFound
+		}
 	}
 
-	project.AppendKey(types.ProjectDeveloperKey(devkey))
+	if key.IsType(types.ProjectKey_DEVELOPER) {
+		ctxBlock := uint64(ctx.BlockHeight())
+
+		// check that the developer key is valid in the current project state
+		var devkeyData types.ProtoDeveloperData
+		found := k.developerKeysFS.FindEntry(ctx, key.Key, ctxBlock, &devkeyData)
+		if !found {
+			// if not found now, check if it is valid in the future (e.g. end of
+			// epoch) state, as it may have been added in this epoch and pending
+			// to become visible).
+			found = k.developerKeysFS.FindEntry(ctx, key.Key, blockHeight, &devkeyData)
+		}
+
+		if !found {
+			return sdkerrors.ErrNotFound
+		}
+
+		// the developer key belongs to a different project
+		if devkeyData.ProjectID != project.GetIndex() {
+			return utils.LavaFormatWarning("failed to unregister key", sdkerrors.ErrNotFound,
+				utils.Attribute{Key: "key", Value: key.Key},
+				utils.Attribute{Key: "keyTypes", Value: key.Kinds},
+			)
+		}
+
+		err := k.developerKeysFS.DelEntry(ctx, key.Key, blockHeight)
+		if err != nil {
+			return err
+		}
+		found = project.DeleteKey(types.ProjectDeveloperKey(key.Key))
+		if !found {
+			panic("unregisterKey: developer key not found")
+		}
+	}
 
 	return nil
 }
diff --git a/x/projects/keeper/keeper.go b/x/projects/keeper/keeper.go
index 8e2b5448ad..caee69523f 100644
--- a/x/projects/keeper/keeper.go
+++ b/x/projects/keeper/keeper.go
@@ -19,7 +19,7 @@ type (
 		memKey     sdk.StoreKey
 		paramstore paramtypes.Subspace
 
-		epochStorageKeeper types.EpochStorageKeeper
+		epochstorageKeeper types.EpochStorageKeeper
 
 		projectsFS      common.FixationStore
 		developerKeysFS common.FixationStore
@@ -31,7 +31,7 @@ func NewKeeper(
 	storeKey,
 	memKey sdk.StoreKey,
 	ps paramtypes.Subspace,
-	epochStorageKeeper types.EpochStorageKeeper,
+	epochstorageKeeper types.EpochStorageKeeper,
 ) *Keeper {
 	// set KeyTable if it has not already been set
 	if !ps.HasKeyTable() {
@@ -48,7 +48,7 @@ func NewKeeper(
 		paramstore:         ps,
 		projectsFS:         *projectsfs,
 		developerKeysFS:    *developerKeysfs,
-		epochStorageKeeper: epochStorageKeeper,
+		epochstorageKeeper: epochstorageKeeper,
 	}
 }
 
diff --git a/x/projects/keeper/project.go b/x/projects/keeper/project.go
index ea9ff53724..843d915cb9 100644
--- a/x/projects/keeper/project.go
+++ b/x/projects/keeper/project.go
@@ -13,7 +13,8 @@ func (k Keeper) GetProjectForBlock(ctx sdk.Context, projectID string, blockHeigh
 	var project types.Project
 
 	if found := k.projectsFS.FindEntry(ctx, projectID, blockHeight, &project); !found {
-		return project, utils.LavaFormatWarning("could not get project for block", fmt.Errorf("project not found"),
+		return project, utils.LavaFormatWarning("failed to get project for block",
+			fmt.Errorf("project or block not found"),
 			utils.Attribute{Key: "project", Value: projectID},
 			utils.Attribute{Key: "blockHeight", Value: blockHeight},
 		)
@@ -23,11 +24,11 @@ func (k Keeper) GetProjectForBlock(ctx sdk.Context, projectID string, blockHeigh
 }
 
 func (k Keeper) GetProjectDeveloperData(ctx sdk.Context, developerKey string, blockHeight uint64) (types.ProtoDeveloperData, error) {
-	var projectDeveloperData types.ProtoDeveloperData
-	if found := k.developerKeysFS.FindEntry(ctx, developerKey, blockHeight, &projectDeveloperData); !found {
-		return types.ProtoDeveloperData{}, fmt.Errorf("GetProjectIDForDeveloper_invalid_key, the requesting key is not registered to a project, developer: %s", developerKey)
+	var devkeyData types.ProtoDeveloperData
+	if found := k.developerKeysFS.FindEntry(ctx, developerKey, blockHeight, &devkeyData); !found {
+		return types.ProtoDeveloperData{}, fmt.Errorf("GetProjectIDForDeveloper_invalid_key: %s", developerKey)
 	}
-	return projectDeveloperData, nil
+	return devkeyData, nil
 }
 
 func (k Keeper) GetProjectForDeveloper(ctx sdk.Context, developerKey string, blockHeight uint64) (proj types.Project, errRet error) {
@@ -48,32 +49,115 @@ func (k Keeper) GetProjectForDeveloper(ctx sdk.Context, developerKey string, blo
 }
 
 func (k Keeper) AddKeysToProject(ctx sdk.Context, projectID string, adminKey string, projectKeys []types.ProjectKey) error {
+	ctxBlock := uint64(ctx.BlockHeight())
+
+	nextEpoch, err := k.epochstorageKeeper.GetNextEpoch(ctx, ctxBlock)
+	if err != nil {
+		return utils.LavaFormatError("AddKeysToProject: failed to get NextEpoch", err,
+			utils.Attribute{Key: "index", Value: projectID},
+		)
+	}
+
+	// validate the admin key against the current project state; but make the
+	// changes on the future (= end of epoch) version if already exists. This
+	// ensures that we do not lose changes if there are e.g. multiple additions
+	// in the same epoch.
+
+	// validate admin key with current state
 	var project types.Project
-	if found := k.projectsFS.FindEntry(ctx, projectID, uint64(ctx.BlockHeight()), &project); !found {
-		return utils.LavaFormatWarning("could not add keys to project", fmt.Errorf("project not found"),
+	if found := k.projectsFS.FindEntry(ctx, projectID, ctxBlock, &project); !found {
+		return utils.LavaFormatWarning("failed to add keys",
+			fmt.Errorf("project not found"),
 			utils.Attribute{Key: "project", Value: projectID},
 		)
 	}
 
-	// check if the admin key is valid
 	if !project.IsAdminKey(adminKey) {
-		return utils.LavaFormatWarning("could not add keys to project", fmt.Errorf("the requesting key is not admin key"),
+		return utils.LavaFormatWarning("failed to add keys",
+			fmt.Errorf("requesting key must be admin key"),
+			utils.Attribute{Key: "project", Value: projectID},
+		)
+	}
+
+	// make changes on the future state
+	project = types.Project{}
+	if found := k.projectsFS.FindEntry(ctx, projectID, nextEpoch, &project); !found {
+		return utils.LavaFormatError("failed to add keys",
+			fmt.Errorf("project not found (for next epoch)"),
 			utils.Attribute{Key: "project", Value: projectID},
 		)
 	}
 
 	for _, projectKey := range projectKeys {
-		err := k.registerKey(ctx, projectKey, &project, uint64(ctx.BlockHeight()))
+		err := k.registerKey(ctx, projectKey, &project, nextEpoch)
 		if err != nil {
 			return utils.LavaFormatError("failed to register key to project", err,
 				utils.Attribute{Key: "project", Value: projectID},
-				utils.Attribute{Key: "key", Value: projectKey.Key},
-				utils.Attribute{Key: "keyTypes", Value: projectKey.Kinds},
 			)
 		}
 	}
 
-	return k.projectsFS.AppendEntry(ctx, projectID, uint64(ctx.BlockHeight()), &project)
+	return k.projectsFS.AppendEntry(ctx, projectID, nextEpoch, &project)
+}
+
+func (k Keeper) DelKeysFromProject(ctx sdk.Context, projectID string, adminKey string, projectKeys []types.ProjectKey) error {
+	ctxBlock := uint64(ctx.BlockHeight())
+
+	nextEpoch, err := k.epochstorageKeeper.GetNextEpoch(ctx, ctxBlock)
+	if err != nil {
+		return utils.LavaFormatError("DelKeysFromProject: failed to get NextEpoch", err,
+			utils.Attribute{Key: "index", Value: projectID},
+		)
+	}
+
+	// validate the admin key against the current project state; but make the
+	// changes on the future (= end of epoch) version if already exists. This
+	// ensures that we do not lose changes if there are e.g. multiple additions
+	// in the same epoch.
+
+	// validate admin key with current state
+	var project types.Project
+	if found := k.projectsFS.FindEntry(ctx, projectID, ctxBlock, &project); !found {
+		return utils.LavaFormatWarning("failed to delete keys",
+			fmt.Errorf("project not found"),
+			utils.Attribute{Key: "project", Value: projectID},
+		)
+	}
+
+	if !project.IsAdminKey(adminKey) {
+		return utils.LavaFormatWarning("failed to delete keys",
+			fmt.Errorf("requesting key must be admin key"),
+			utils.Attribute{Key: "project", Value: projectID},
+		)
+	}
+
+	project = types.Project{}
+	if found := k.projectsFS.FindEntry(ctx, projectID, nextEpoch, &project); !found {
+		return utils.LavaFormatWarning("failed to delete keys",
+			fmt.Errorf("project not found (for next epoch)"),
+			utils.Attribute{Key: "project", Value: projectID},
+		)
+	}
+
+	// make changes on the future state
+	for _, projectKey := range projectKeys {
+		// check if the deleted key is subscription owner
+		if projectKey.Key == project.Subscription {
+			return utils.LavaFormatWarning("failed to delete keys",
+				fmt.Errorf("subscription key may not be deleted"),
+				utils.Attribute{Key: "project", Value: projectID},
+			)
+		}
+
+		err := k.unregisterKey(ctx, projectKey, &project, nextEpoch)
+		if err != nil {
+			return utils.LavaFormatWarning("failed to unregister key to project", err,
+				utils.Attribute{Key: "project", Value: projectID},
+			)
+		}
+	}
+
+	return k.projectsFS.AppendEntry(ctx, projectID, nextEpoch, &project)
 }
 
 func (k Keeper) ChargeComputeUnitsToProject(ctx sdk.Context, project types.Project, blockHeight uint64, cu uint64) (err error) {
@@ -123,7 +207,7 @@ func (k Keeper) SetProjectPolicy(ctx sdk.Context, projectIDs []string, policy *t
 			}
 		}
 
-		nextEpoch, err := k.epochStorageKeeper.GetNextEpoch(ctx, uint64(ctx.BlockHeight()))
+		nextEpoch, err := k.epochstorageKeeper.GetNextEpoch(ctx, uint64(ctx.BlockHeight()))
 		if err != nil {
 			panic("could not set policy. can't get next epoch")
 		}
diff --git a/x/projects/types/project.go b/x/projects/types/project.go
index 4de99219df..36b1bae1a3 100644
--- a/x/projects/types/project.go
+++ b/x/projects/types/project.go
@@ -81,14 +81,32 @@ func (project *Project) GetKey(key string) ProjectKey {
 	return ProjectKey{}
 }
 
-func (project *Project) AppendKey(key ProjectKey) {
+func (project *Project) AppendKey(key ProjectKey) bool {
 	for i, projectKey := range project.ProjectKeys {
 		if projectKey.Key == key.Key {
 			project.ProjectKeys[i].Kinds |= key.Kinds
-			return
+			return true
 		}
 	}
 	project.ProjectKeys = append(project.ProjectKeys, key)
+	return false
+}
+
+func (project *Project) DeleteKey(key ProjectKey) bool {
+	length := len(project.ProjectKeys)
+	for i, projectKey := range project.ProjectKeys {
+		if projectKey.Key == key.Key {
+			project.ProjectKeys[i].Kinds &= ^key.Kinds
+			if project.ProjectKeys[i].Kinds == uint32(ProjectKey_NONE) {
+				if i < length-1 {
+					project.ProjectKeys[i] = project.ProjectKeys[length-1]
+				}
+				project.ProjectKeys = project.ProjectKeys[0 : length-1]
+			}
+			return true
+		}
+	}
+	return false
 }
 
 func (project *Project) IsAdminKey(key string) bool {

From d33272bb032842a6c7df5a65559c05eb25e11d78 Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sat, 3 Jun 2023 23:33:58 +0800
Subject: [PATCH 04/10] CNS-420: support tx project del-keys command

---
 proto/projects/project.proto                  |   2 +-
 proto/projects/tx.proto                       |  10 +
 x/projects/client/cli/tx.go                   |   1 +
 x/projects/client/cli/tx_add_keys.go          |   5 -
 x/projects/client/cli/tx_del_keys.go          |  74 +++
 x/projects/handler.go                         |   3 +
 .../keeper/msg_server_del_project_keys.go     |  32 ++
 x/projects/module_simulation.go               |  15 +
 x/projects/simulation/del_project_keys.go     |  27 +
 x/projects/types/codec.go                     |   4 +
 x/projects/types/message_del_keys.go          |  48 ++
 x/projects/types/message_del_keys_test.go     |  40 ++
 x/projects/types/tx.pb.go                     | 480 +++++++++++++++++-
 13 files changed, 715 insertions(+), 26 deletions(-)
 create mode 100644 x/projects/client/cli/tx_del_keys.go
 create mode 100644 x/projects/keeper/msg_server_del_project_keys.go
 create mode 100644 x/projects/simulation/del_project_keys.go
 create mode 100644 x/projects/types/message_del_keys.go
 create mode 100644 x/projects/types/message_del_keys_test.go

diff --git a/proto/projects/project.proto b/proto/projects/project.proto
index 620f1650f1..148e6abacd 100644
--- a/proto/projects/project.proto
+++ b/proto/projects/project.proto
@@ -29,7 +29,7 @@ message ProjectKey {
         DEVELOPER = 0x2;
     }
 
-	uint32 kinds = 4;
+    uint32 kinds = 4;
 }
 
 // protobuf expected in YAML format: used "moretags" to simplify parsing
diff --git a/proto/projects/tx.proto b/proto/projects/tx.proto
index a67b94bd66..8c62bd7403 100644
--- a/proto/projects/tx.proto
+++ b/proto/projects/tx.proto
@@ -11,6 +11,7 @@ option go_package = "github.com/lavanet/lava/x/projects/types";
 // Msg defines the Msg service.
 service Msg {
   rpc AddKeys(MsgAddKeys) returns (MsgAddKeysResponse);
+  rpc DelKeys(MsgDelKeys) returns (MsgDelKeysResponse);
   rpc SetPolicy(MsgSetPolicy) returns (MsgSetPolicyResponse);
   rpc SetSubscriptionPolicy(MsgSetSubscriptionPolicy) returns (MsgSetSubscriptionPolicyResponse);
 // this line is used by starport scaffolding # proto/tx/rpc
@@ -25,6 +26,15 @@ message MsgAddKeys {
 message MsgAddKeysResponse {
 }
 
+message MsgDelKeys {
+  string creator = 1;
+  string project = 2;
+  repeated ProjectKey project_keys = 3 [(gogoproto.nullable) = false];
+}
+
+message MsgDelKeysResponse {
+}
+
 message MsgSetPolicy {
   string creator = 1;
   string project = 2;
diff --git a/x/projects/client/cli/tx.go b/x/projects/client/cli/tx.go
index d9bd893608..895d6dbd18 100644
--- a/x/projects/client/cli/tx.go
+++ b/x/projects/client/cli/tx.go
@@ -29,6 +29,7 @@ func GetTxCmd() *cobra.Command {
 	}
 
 	cmd.AddCommand(CmdAddKeys())
+	cmd.AddCommand(CmdDelKeys())
 	cmd.AddCommand(CmdSetPolicy())
 	cmd.AddCommand(CmdSetSubscriptionPolicy())
 	// this line is used by starport scaffolding # 1
diff --git a/x/projects/client/cli/tx_add_keys.go b/x/projects/client/cli/tx_add_keys.go
index 75b1998c92..f3b1ef3c86 100644
--- a/x/projects/client/cli/tx_add_keys.go
+++ b/x/projects/client/cli/tx_add_keys.go
@@ -1,8 +1,6 @@
 package cli
 
 import (
-	"strconv"
-
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	"github.com/cosmos/cosmos-sdk/client/tx"
@@ -11,15 +9,12 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var _ = strconv.Itoa(0)
-
 func CmdAddKeys() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "add-keys [project-id] [optional: project-keys-file-path]",
 		Short: "Add developer/admin keys to an existing project",
 		Long: `The add-keys command allows the project admin to add new project keys (admin/developer) to the project.
 		To add the keys you can optionally provide a YAML file of the new project keys (see example in cookbook/project/example_project_keys.yml).
-		Note that in project keys, to define the key type, you should follow the enum described in the top of example_project_keys.yml.
 		Another way to add keys is with the --admin-key and --developer-key flags.`,
 		Example: `required flags: --from <admin-key> (the project's subscription address is also considered admin)
 				  
diff --git a/x/projects/client/cli/tx_del_keys.go b/x/projects/client/cli/tx_del_keys.go
new file mode 100644
index 0000000000..bc8dcc263c
--- /dev/null
+++ b/x/projects/client/cli/tx_del_keys.go
@@ -0,0 +1,74 @@
+package cli
+
+import (
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/cosmos/cosmos-sdk/client/flags"
+	"github.com/cosmos/cosmos-sdk/client/tx"
+	commontypes "github.com/lavanet/lava/common/types"
+	"github.com/lavanet/lava/x/projects/types"
+	"github.com/spf13/cobra"
+)
+
+func CmdDelKeys() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "del-keys [project-id] [optional: project-keys-file-path]",
+		Short: "Delete developer/admin keys to an existing project",
+		Long: `The del-keys command allows the project admin to delete project keys (admin/developer) from the project.
+		To delete the keys you can optionally provide a YAML file of the project keys to delete (see example in cookbook/project/example_project_keys.yml).
+		Another way to delete keys is with the --admin-key and --developer-key flags.`,
+		Example: `required flags: --from <admin-key> (the project's subscription address is also considered admin)
+
+		lavad tx project del-keys [project-id] [project-keys-file-path] --from <admin-key>
+		lavad tx project del-keys [project-id] --admin-key <other-admin-key> --admin-key <another-admin-key> --developer-key <developer-key> --from <admin-key>`,
+		Args: cobra.RangeArgs(1, 2),
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			projectID := args[0]
+			var projectKeys []types.ProjectKey
+
+			if len(args) > 1 {
+				projectKeysFilePath := args[1]
+				err = commontypes.ReadYaml(projectKeysFilePath, "Project-Keys", &projectKeys)
+				if err != nil {
+					return err
+				}
+			} else {
+				developerFlagsValue, err := cmd.Flags().GetStringSlice("developer-key")
+				if err != nil {
+					return err
+				}
+				for _, developerFlagValue := range developerFlagsValue {
+					projectKeys = append(projectKeys, types.ProjectDeveloperKey(developerFlagValue))
+				}
+
+				adminAddresses, err := cmd.Flags().GetStringSlice("admin-key")
+				if err != nil {
+					return err
+				}
+				for _, adminAddress := range adminAddresses {
+					projectKeys = append(projectKeys, types.ProjectAdminKey(adminAddress))
+				}
+			}
+
+			clientCtx, err := client.GetClientTxContext(cmd)
+			if err != nil {
+				return err
+			}
+
+			msg := types.NewMsgDelKeys(
+				clientCtx.GetFromAddress().String(),
+				projectID,
+				projectKeys,
+			)
+			if err := msg.ValidateBasic(); err != nil {
+				return err
+			}
+			return tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg)
+		},
+	}
+
+	cmd.Flags().StringSlice("developer-key", []string{}, "Developer keys to add")
+	cmd.Flags().StringSlice("admin-key", []string{}, "Admin keys to add")
+	flags.AddTxFlagsToCmd(cmd)
+
+	return cmd
+}
diff --git a/x/projects/handler.go b/x/projects/handler.go
index 7df3cee316..0f98bc68fe 100644
--- a/x/projects/handler.go
+++ b/x/projects/handler.go
@@ -20,6 +20,9 @@ func NewHandler(k keeper.Keeper) sdk.Handler {
 		case *types.MsgAddKeys:
 			res, err := msgServer.AddKeys(sdk.WrapSDKContext(ctx), msg)
 			return sdk.WrapServiceResult(ctx, res, err)
+		case *types.MsgDelKeys:
+			res, err := msgServer.DelKeys(sdk.WrapSDKContext(ctx), msg)
+			return sdk.WrapServiceResult(ctx, res, err)
 		case *types.MsgSetPolicy:
 			res, err := msgServer.SetPolicy(sdk.WrapSDKContext(ctx), msg)
 			return sdk.WrapServiceResult(ctx, res, err)
diff --git a/x/projects/keeper/msg_server_del_project_keys.go b/x/projects/keeper/msg_server_del_project_keys.go
new file mode 100644
index 0000000000..ee88477b1b
--- /dev/null
+++ b/x/projects/keeper/msg_server_del_project_keys.go
@@ -0,0 +1,32 @@
+package keeper
+
+import (
+	"context"
+	"fmt"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/lavanet/lava/utils"
+	"github.com/lavanet/lava/x/projects/types"
+)
+
+func (k msgServer) DelKeys(goCtx context.Context, msg *types.MsgDelKeys) (*types.MsgDelKeysResponse, error) {
+	ctx := sdk.UnwrapSDKContext(goCtx)
+
+	for _, projectKey := range msg.GetProjectKeys() {
+		if !projectKey.IsTypeValid() {
+			return nil, utils.LavaFormatWarning(
+				"invalid project key type (must be ADMIN(=1) or DEVELOPER(=2)",
+				fmt.Errorf("invalid project key type"),
+				utils.Attribute{Key: "key", Value: projectKey.Key},
+				utils.Attribute{Key: "keyType", Value: projectKey.Kinds},
+			)
+		}
+	}
+
+	err := k.DelKeysFromProject(ctx, msg.Project, msg.Creator, msg.ProjectKeys)
+	if err != nil {
+		return nil, err
+	}
+
+	return &types.MsgDelKeysResponse{}, nil
+}
diff --git a/x/projects/module_simulation.go b/x/projects/module_simulation.go
index a19045ded7..e593edc93d 100644
--- a/x/projects/module_simulation.go
+++ b/x/projects/module_simulation.go
@@ -28,6 +28,10 @@ const (
 	// TODO: Determine the simulation weight value
 	defaultWeightMsgAddKeys int = 100
 
+	opWeightMsgDelKeys = "op_weight_msg_del_keys"
+	// TODO: Determine the simulation weight value
+	defaultWeightMsgDelKeys int = 100
+
 	opWeightMsgSetPolicy = "op_weight_msg_set_admin_policy"
 	// TODO: Determine the simulation weight value
 	defaultWeightMsgSetPolicy int = 100
@@ -80,6 +84,17 @@ func (am AppModule) WeightedOperations(simState module.SimulationState) []simtyp
 		projectssimulation.SimulateMsgAddKeys(am.keeper),
 	))
 
+	var weightMsgDelKeys int
+	simState.AppParams.GetOrGenerate(simState.Cdc, opWeightMsgDelKeys, &weightMsgDelKeys, nil,
+		func(_ *rand.Rand) {
+			weightMsgDelKeys = defaultWeightMsgDelKeys
+		},
+	)
+	operations = append(operations, simulation.NewWeightedOperation(
+		weightMsgDelKeys,
+		projectssimulation.SimulateMsgDelKeys(am.keeper),
+	))
+
 	var weightMsgSetPolicy int
 	simState.AppParams.GetOrGenerate(simState.Cdc, opWeightMsgSetPolicy, &weightMsgSetPolicy, nil,
 		func(_ *rand.Rand) {
diff --git a/x/projects/simulation/del_project_keys.go b/x/projects/simulation/del_project_keys.go
new file mode 100644
index 0000000000..d9a87c18fe
--- /dev/null
+++ b/x/projects/simulation/del_project_keys.go
@@ -0,0 +1,27 @@
+package simulation
+
+import (
+	"math/rand"
+
+	"github.com/cosmos/cosmos-sdk/baseapp"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
+	"github.com/lavanet/lava/x/projects/keeper"
+	"github.com/lavanet/lava/x/projects/types"
+)
+
+func SimulateMsgDelKeys(
+	k keeper.Keeper,
+) simtypes.Operation {
+	return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string,
+	) (simtypes.OperationMsg, []simtypes.FutureOperation, error) {
+		simAccount, _ := simtypes.RandomAcc(r, accs)
+		msg := &types.MsgDelKeys{
+			Creator: simAccount.Address.String(),
+		}
+
+		// TODO: Handling the DelKeys simulation
+
+		return simtypes.NoOpMsg(types.ModuleName, msg.Type(), "DelKeys simulation not implemented"), nil, nil
+	}
+}
diff --git a/x/projects/types/codec.go b/x/projects/types/codec.go
index 20f09c6b92..c6f25bfddf 100644
--- a/x/projects/types/codec.go
+++ b/x/projects/types/codec.go
@@ -10,6 +10,7 @@ import (
 
 func RegisterCodec(cdc *codec.LegacyAmino) {
 	cdc.RegisterConcrete(&MsgAddKeys{}, "projects/AddKeys", nil)
+	cdc.RegisterConcrete(&MsgDelKeys{}, "projects/DelKeys", nil)
 	cdc.RegisterConcrete(&MsgSetPolicy{}, "projects/SetPolicy", nil)
 	cdc.RegisterConcrete(&MsgSetSubscriptionPolicy{}, "projects/SetSubscriptionPolicy", nil)
 	// this line is used by starport scaffolding # 2
@@ -19,6 +20,9 @@ func RegisterInterfaces(registry cdctypes.InterfaceRegistry) {
 	registry.RegisterImplementations((*sdk.Msg)(nil),
 		&MsgAddKeys{},
 	)
+	registry.RegisterImplementations((*sdk.Msg)(nil),
+		&MsgDelKeys{},
+	)
 	registry.RegisterImplementations((*sdk.Msg)(nil),
 		&MsgSetPolicy{},
 	)
diff --git a/x/projects/types/message_del_keys.go b/x/projects/types/message_del_keys.go
new file mode 100644
index 0000000000..1f9ce022b1
--- /dev/null
+++ b/x/projects/types/message_del_keys.go
@@ -0,0 +1,48 @@
+package types
+
+import (
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
+)
+
+const TypeMsgDelKeys = "del_keys"
+
+var _ sdk.Msg = &MsgDelKeys{}
+
+func NewMsgDelKeys(creator string, projectID string, projectKeys []ProjectKey) *MsgDelKeys {
+	return &MsgDelKeys{
+		Creator:     creator,
+		Project:     projectID,
+		ProjectKeys: projectKeys,
+	}
+}
+
+func (msg *MsgDelKeys) Route() string {
+	return RouterKey
+}
+
+func (msg *MsgDelKeys) Type() string {
+	return TypeMsgDelKeys
+}
+
+func (msg *MsgDelKeys) GetSigners() []sdk.AccAddress {
+	creator, err := sdk.AccAddressFromBech32(msg.Creator)
+	if err != nil {
+		panic(err)
+	}
+	return []sdk.AccAddress{creator}
+}
+
+func (msg *MsgDelKeys) GetSignBytes() []byte {
+	bz := ModuleCdc.MustMarshalJSON(msg)
+	return sdk.MustSortJSON(bz)
+}
+
+func (msg *MsgDelKeys) ValidateBasic() error {
+	_, err := sdk.AccAddressFromBech32(msg.Creator)
+	if err != nil {
+		return sdkerrors.Wrapf(sdkerrors.ErrInvalidAddress, "invalid creator address (%s)", err)
+	}
+
+	return nil
+}
diff --git a/x/projects/types/message_del_keys_test.go b/x/projects/types/message_del_keys_test.go
new file mode 100644
index 0000000000..9f92be4cc6
--- /dev/null
+++ b/x/projects/types/message_del_keys_test.go
@@ -0,0 +1,40 @@
+package types
+
+import (
+	"testing"
+
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
+	"github.com/lavanet/lava/testutil/sample"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMsgDelKeys_ValidateBasic(t *testing.T) {
+	tests := []struct {
+		name string
+		msg  MsgDelKeys
+		err  error
+	}{
+		{
+			name: "invalid address",
+			msg: MsgDelKeys{
+				Creator: "invalid_address",
+			},
+			err: sdkerrors.ErrInvalidAddress,
+		}, {
+			name: "valid address",
+			msg: MsgDelKeys{
+				Creator: sample.AccAddress(),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.msg.ValidateBasic()
+			if tt.err != nil {
+				require.ErrorIs(t, err, tt.err)
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}
diff --git a/x/projects/types/tx.pb.go b/x/projects/types/tx.pb.go
index fca8629b46..abb7be9e39 100644
--- a/x/projects/types/tx.pb.go
+++ b/x/projects/types/tx.pb.go
@@ -124,6 +124,102 @@ func (m *MsgAddKeysResponse) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_MsgAddKeysResponse proto.InternalMessageInfo
 
+type MsgDelKeys struct {
+	Creator     string       `protobuf:"bytes,1,opt,name=creator,proto3" json:"creator,omitempty"`
+	Project     string       `protobuf:"bytes,2,opt,name=project,proto3" json:"project,omitempty"`
+	ProjectKeys []ProjectKey `protobuf:"bytes,3,rep,name=project_keys,json=projectKeys,proto3" json:"project_keys"`
+}
+
+func (m *MsgDelKeys) Reset()         { *m = MsgDelKeys{} }
+func (m *MsgDelKeys) String() string { return proto.CompactTextString(m) }
+func (*MsgDelKeys) ProtoMessage()    {}
+func (*MsgDelKeys) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b5dcbe7dfba713c0, []int{2}
+}
+func (m *MsgDelKeys) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *MsgDelKeys) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_MsgDelKeys.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *MsgDelKeys) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MsgDelKeys.Merge(m, src)
+}
+func (m *MsgDelKeys) XXX_Size() int {
+	return m.Size()
+}
+func (m *MsgDelKeys) XXX_DiscardUnknown() {
+	xxx_messageInfo_MsgDelKeys.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MsgDelKeys proto.InternalMessageInfo
+
+func (m *MsgDelKeys) GetCreator() string {
+	if m != nil {
+		return m.Creator
+	}
+	return ""
+}
+
+func (m *MsgDelKeys) GetProject() string {
+	if m != nil {
+		return m.Project
+	}
+	return ""
+}
+
+func (m *MsgDelKeys) GetProjectKeys() []ProjectKey {
+	if m != nil {
+		return m.ProjectKeys
+	}
+	return nil
+}
+
+type MsgDelKeysResponse struct {
+}
+
+func (m *MsgDelKeysResponse) Reset()         { *m = MsgDelKeysResponse{} }
+func (m *MsgDelKeysResponse) String() string { return proto.CompactTextString(m) }
+func (*MsgDelKeysResponse) ProtoMessage()    {}
+func (*MsgDelKeysResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b5dcbe7dfba713c0, []int{3}
+}
+func (m *MsgDelKeysResponse) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *MsgDelKeysResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_MsgDelKeysResponse.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *MsgDelKeysResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MsgDelKeysResponse.Merge(m, src)
+}
+func (m *MsgDelKeysResponse) XXX_Size() int {
+	return m.Size()
+}
+func (m *MsgDelKeysResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_MsgDelKeysResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MsgDelKeysResponse proto.InternalMessageInfo
+
 type MsgSetPolicy struct {
 	Creator string `protobuf:"bytes,1,opt,name=creator,proto3" json:"creator,omitempty"`
 	Project string `protobuf:"bytes,2,opt,name=project,proto3" json:"project,omitempty"`
@@ -134,7 +230,7 @@ func (m *MsgSetPolicy) Reset()         { *m = MsgSetPolicy{} }
 func (m *MsgSetPolicy) String() string { return proto.CompactTextString(m) }
 func (*MsgSetPolicy) ProtoMessage()    {}
 func (*MsgSetPolicy) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b5dcbe7dfba713c0, []int{2}
+	return fileDescriptor_b5dcbe7dfba713c0, []int{4}
 }
 func (m *MsgSetPolicy) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -191,7 +287,7 @@ func (m *MsgSetPolicyResponse) Reset()         { *m = MsgSetPolicyResponse{} }
 func (m *MsgSetPolicyResponse) String() string { return proto.CompactTextString(m) }
 func (*MsgSetPolicyResponse) ProtoMessage()    {}
 func (*MsgSetPolicyResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b5dcbe7dfba713c0, []int{3}
+	return fileDescriptor_b5dcbe7dfba713c0, []int{5}
 }
 func (m *MsgSetPolicyResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -230,7 +326,7 @@ func (m *MsgSetSubscriptionPolicy) Reset()         { *m = MsgSetSubscriptionPoli
 func (m *MsgSetSubscriptionPolicy) String() string { return proto.CompactTextString(m) }
 func (*MsgSetSubscriptionPolicy) ProtoMessage()    {}
 func (*MsgSetSubscriptionPolicy) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b5dcbe7dfba713c0, []int{4}
+	return fileDescriptor_b5dcbe7dfba713c0, []int{6}
 }
 func (m *MsgSetSubscriptionPolicy) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -287,7 +383,7 @@ func (m *MsgSetSubscriptionPolicyResponse) Reset()         { *m = MsgSetSubscrip
 func (m *MsgSetSubscriptionPolicyResponse) String() string { return proto.CompactTextString(m) }
 func (*MsgSetSubscriptionPolicyResponse) ProtoMessage()    {}
 func (*MsgSetSubscriptionPolicyResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b5dcbe7dfba713c0, []int{5}
+	return fileDescriptor_b5dcbe7dfba713c0, []int{7}
 }
 func (m *MsgSetSubscriptionPolicyResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -319,6 +415,8 @@ var xxx_messageInfo_MsgSetSubscriptionPolicyResponse proto.InternalMessageInfo
 func init() {
 	proto.RegisterType((*MsgAddKeys)(nil), "lavanet.lava.projects.MsgAddKeys")
 	proto.RegisterType((*MsgAddKeysResponse)(nil), "lavanet.lava.projects.MsgAddKeysResponse")
+	proto.RegisterType((*MsgDelKeys)(nil), "lavanet.lava.projects.MsgDelKeys")
+	proto.RegisterType((*MsgDelKeysResponse)(nil), "lavanet.lava.projects.MsgDelKeysResponse")
 	proto.RegisterType((*MsgSetPolicy)(nil), "lavanet.lava.projects.MsgSetPolicy")
 	proto.RegisterType((*MsgSetPolicyResponse)(nil), "lavanet.lava.projects.MsgSetPolicyResponse")
 	proto.RegisterType((*MsgSetSubscriptionPolicy)(nil), "lavanet.lava.projects.MsgSetSubscriptionPolicy")
@@ -328,7 +426,7 @@ func init() {
 func init() { proto.RegisterFile("projects/tx.proto", fileDescriptor_b5dcbe7dfba713c0) }
 
 var fileDescriptor_b5dcbe7dfba713c0 = []byte{
-	// 394 bytes of a gzipped FileDescriptorProto
+	// 420 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2c, 0x28, 0xca, 0xcf,
 	0x4a, 0x4d, 0x2e, 0x29, 0xd6, 0x2f, 0xa9, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0xcd,
 	0x49, 0x2c, 0x4b, 0xcc, 0x4b, 0x2d, 0xd1, 0x03, 0xd1, 0x7a, 0x30, 0x79, 0x29, 0x31, 0xb8, 0x4a,
@@ -339,21 +437,23 @@ var fileDescriptor_b5dcbe7dfba713c0 = []byte{
 	0x81, 0x32, 0xe3, 0xb3, 0x53, 0x2b, 0x8b, 0x25, 0x98, 0x15, 0x98, 0x35, 0xb8, 0x8d, 0x14, 0xf5,
 	0xb0, 0x3a, 0x4f, 0x2f, 0x00, 0xc2, 0xf0, 0x4e, 0xad, 0x74, 0x62, 0x39, 0x71, 0x4f, 0x9e, 0x21,
 	0x88, 0xbb, 0x00, 0x2e, 0x52, 0xac, 0x24, 0xc2, 0x25, 0x84, 0x70, 0x4d, 0x50, 0x6a, 0x71, 0x41,
-	0x7e, 0x5e, 0x71, 0xaa, 0x52, 0x3d, 0x17, 0x8f, 0x6f, 0x71, 0x7a, 0x70, 0x6a, 0x49, 0x40, 0x7e,
-	0x4e, 0x66, 0x72, 0x25, 0x59, 0xae, 0xb4, 0xe6, 0x62, 0x2b, 0x00, 0xeb, 0x96, 0x60, 0x56, 0x60,
-	0xd4, 0xe0, 0x36, 0x92, 0xc5, 0xe5, 0x3e, 0xb0, 0x22, 0xa8, 0xdb, 0xa0, 0x5a, 0x94, 0xc4, 0xb8,
-	0x44, 0x90, 0x1d, 0x00, 0x77, 0x58, 0x2f, 0x23, 0x97, 0x04, 0x44, 0x22, 0xb8, 0x34, 0xa9, 0x38,
-	0xb9, 0x28, 0xb3, 0xa0, 0x24, 0x33, 0x3f, 0x8f, 0xa0, 0x2b, 0xa5, 0xb8, 0x38, 0x60, 0xf6, 0x49,
-	0x30, 0x29, 0x30, 0x6b, 0x70, 0x06, 0xc1, 0xf9, 0x94, 0xb9, 0x53, 0x89, 0x4b, 0x01, 0x97, 0x73,
-	0x60, 0x6e, 0x36, 0x3a, 0xc8, 0xc4, 0xc5, 0xec, 0x5b, 0x9c, 0x2e, 0x14, 0xce, 0xc5, 0x0e, 0x8b,
-	0x75, 0x5c, 0x71, 0x85, 0x88, 0x0a, 0x29, 0x4d, 0x82, 0x4a, 0x60, 0x16, 0x08, 0xc5, 0x72, 0x71,
-	0x22, 0xa2, 0x4a, 0x19, 0xb7, 0x3e, 0xb8, 0x22, 0x29, 0x6d, 0x22, 0x14, 0xc1, 0x8d, 0x6f, 0x64,
-	0xe4, 0x12, 0xc5, 0x1e, 0xe0, 0xfa, 0x78, 0x8d, 0xc1, 0xd4, 0x20, 0x65, 0x4e, 0xa2, 0x06, 0x98,
-	0x1b, 0x9c, 0x9c, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6,
-	0x09, 0x8f, 0xe5, 0x18, 0x2e, 0x3c, 0x96, 0x63, 0xb8, 0xf1, 0x58, 0x8e, 0x21, 0x4a, 0x23, 0x3d,
-	0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0x6a, 0x38, 0x98, 0xd6, 0xaf, 0xd0,
-	0x47, 0xe4, 0xe0, 0xca, 0x82, 0xd4, 0xe2, 0x24, 0x36, 0x70, 0x06, 0x34, 0x06, 0x04, 0x00, 0x00,
-	0xff, 0xff, 0x4b, 0x75, 0x92, 0x3b, 0xda, 0x03, 0x00, 0x00,
+	0x7e, 0x5e, 0x71, 0x2a, 0xcc, 0x91, 0x2e, 0xa9, 0x39, 0x83, 0xc8, 0x91, 0x50, 0xd7, 0xc0, 0x1d,
+	0x59, 0xcf, 0xc5, 0xe3, 0x5b, 0x9c, 0x1e, 0x9c, 0x5a, 0x12, 0x90, 0x9f, 0x93, 0x99, 0x5c, 0x49,
+	0x96, 0x2b, 0xad, 0xb9, 0xd8, 0x0a, 0xc0, 0xba, 0x25, 0x98, 0x15, 0x18, 0x35, 0xb8, 0x8d, 0x64,
+	0x71, 0xb9, 0x0f, 0xac, 0x08, 0xea, 0x36, 0xa8, 0x16, 0x25, 0x31, 0x2e, 0x11, 0x64, 0x07, 0xc0,
+	0x1d, 0xd6, 0xcb, 0xc8, 0x25, 0x01, 0x91, 0x08, 0x2e, 0x4d, 0x2a, 0x4e, 0x2e, 0xca, 0x2c, 0x28,
+	0xc9, 0xcc, 0xcf, 0x23, 0xe8, 0x4a, 0x29, 0x2e, 0x0e, 0x98, 0x7d, 0x12, 0x4c, 0x0a, 0xcc, 0x1a,
+	0x9c, 0x41, 0x70, 0x3e, 0x65, 0xee, 0x54, 0xe2, 0x52, 0xc0, 0xe5, 0x1c, 0x98, 0x9b, 0x8d, 0x66,
+	0x31, 0x73, 0x31, 0xfb, 0x16, 0xa7, 0x0b, 0x85, 0x73, 0xb1, 0xc3, 0x92, 0x26, 0xae, 0xb8, 0x42,
+	0xa4, 0x17, 0x29, 0x4d, 0x82, 0x4a, 0x60, 0x16, 0x80, 0x0c, 0x86, 0x25, 0x27, 0x3c, 0x06, 0x43,
+	0x95, 0xe0, 0x33, 0x18, 0x2d, 0x19, 0x08, 0xc5, 0x72, 0x71, 0x22, 0xd2, 0x80, 0x32, 0x6e, 0x7d,
+	0x70, 0x45, 0x52, 0xda, 0x44, 0x28, 0x82, 0x1b, 0xdf, 0xc8, 0xc8, 0x25, 0x8a, 0x3d, 0x26, 0xf5,
+	0xf1, 0x1a, 0x83, 0xa9, 0x41, 0xca, 0x9c, 0x44, 0x0d, 0x30, 0x37, 0x38, 0x39, 0x9d, 0x78, 0x24,
+	0xc7, 0x78, 0xe1, 0x91, 0x1c, 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x13, 0x1e, 0xcb, 0x31, 0x5c, 0x78,
+	0x2c, 0xc7, 0x70, 0xe3, 0xb1, 0x1c, 0x43, 0x94, 0x46, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e,
+	0x72, 0x7e, 0xae, 0x3e, 0xd4, 0x70, 0x30, 0xad, 0x5f, 0xa1, 0x8f, 0x28, 0xbf, 0x2a, 0x0b, 0x52,
+	0x8b, 0x93, 0xd8, 0xc0, 0xc5, 0x8f, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x26, 0xb0, 0x01, 0xc4,
+	0xd8, 0x04, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -369,6 +469,7 @@ const _ = grpc.SupportPackageIsVersion4
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
 type MsgClient interface {
 	AddKeys(ctx context.Context, in *MsgAddKeys, opts ...grpc.CallOption) (*MsgAddKeysResponse, error)
+	DelKeys(ctx context.Context, in *MsgDelKeys, opts ...grpc.CallOption) (*MsgDelKeysResponse, error)
 	SetPolicy(ctx context.Context, in *MsgSetPolicy, opts ...grpc.CallOption) (*MsgSetPolicyResponse, error)
 	SetSubscriptionPolicy(ctx context.Context, in *MsgSetSubscriptionPolicy, opts ...grpc.CallOption) (*MsgSetSubscriptionPolicyResponse, error)
 }
@@ -390,6 +491,15 @@ func (c *msgClient) AddKeys(ctx context.Context, in *MsgAddKeys, opts ...grpc.Ca
 	return out, nil
 }
 
+func (c *msgClient) DelKeys(ctx context.Context, in *MsgDelKeys, opts ...grpc.CallOption) (*MsgDelKeysResponse, error) {
+	out := new(MsgDelKeysResponse)
+	err := c.cc.Invoke(ctx, "/lavanet.lava.projects.Msg/DelKeys", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *msgClient) SetPolicy(ctx context.Context, in *MsgSetPolicy, opts ...grpc.CallOption) (*MsgSetPolicyResponse, error) {
 	out := new(MsgSetPolicyResponse)
 	err := c.cc.Invoke(ctx, "/lavanet.lava.projects.Msg/SetPolicy", in, out, opts...)
@@ -411,6 +521,7 @@ func (c *msgClient) SetSubscriptionPolicy(ctx context.Context, in *MsgSetSubscri
 // MsgServer is the server API for Msg service.
 type MsgServer interface {
 	AddKeys(context.Context, *MsgAddKeys) (*MsgAddKeysResponse, error)
+	DelKeys(context.Context, *MsgDelKeys) (*MsgDelKeysResponse, error)
 	SetPolicy(context.Context, *MsgSetPolicy) (*MsgSetPolicyResponse, error)
 	SetSubscriptionPolicy(context.Context, *MsgSetSubscriptionPolicy) (*MsgSetSubscriptionPolicyResponse, error)
 }
@@ -422,6 +533,9 @@ type UnimplementedMsgServer struct {
 func (*UnimplementedMsgServer) AddKeys(ctx context.Context, req *MsgAddKeys) (*MsgAddKeysResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method AddKeys not implemented")
 }
+func (*UnimplementedMsgServer) DelKeys(ctx context.Context, req *MsgDelKeys) (*MsgDelKeysResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DelKeys not implemented")
+}
 func (*UnimplementedMsgServer) SetPolicy(ctx context.Context, req *MsgSetPolicy) (*MsgSetPolicyResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method SetPolicy not implemented")
 }
@@ -451,6 +565,24 @@ func _Msg_AddKeys_Handler(srv interface{}, ctx context.Context, dec func(interfa
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Msg_DelKeys_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(MsgDelKeys)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MsgServer).DelKeys(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/lavanet.lava.projects.Msg/DelKeys",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MsgServer).DelKeys(ctx, req.(*MsgDelKeys))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Msg_SetPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(MsgSetPolicy)
 	if err := dec(in); err != nil {
@@ -495,6 +627,10 @@ var _Msg_serviceDesc = grpc.ServiceDesc{
 			MethodName: "AddKeys",
 			Handler:    _Msg_AddKeys_Handler,
 		},
+		{
+			MethodName: "DelKeys",
+			Handler:    _Msg_DelKeys_Handler,
+		},
 		{
 			MethodName: "SetPolicy",
 			Handler:    _Msg_SetPolicy_Handler,
@@ -582,6 +718,80 @@ func (m *MsgAddKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *MsgDelKeys) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *MsgDelKeys) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *MsgDelKeys) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.ProjectKeys) > 0 {
+		for iNdEx := len(m.ProjectKeys) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.ProjectKeys[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTx(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x1a
+		}
+	}
+	if len(m.Project) > 0 {
+		i -= len(m.Project)
+		copy(dAtA[i:], m.Project)
+		i = encodeVarintTx(dAtA, i, uint64(len(m.Project)))
+		i--
+		dAtA[i] = 0x12
+	}
+	if len(m.Creator) > 0 {
+		i -= len(m.Creator)
+		copy(dAtA[i:], m.Creator)
+		i = encodeVarintTx(dAtA, i, uint64(len(m.Creator)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *MsgDelKeysResponse) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *MsgDelKeysResponse) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *MsgDelKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	return len(dAtA) - i, nil
+}
+
 func (m *MsgSetPolicy) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -767,6 +977,38 @@ func (m *MsgAddKeysResponse) Size() (n int) {
 	return n
 }
 
+func (m *MsgDelKeys) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Creator)
+	if l > 0 {
+		n += 1 + l + sovTx(uint64(l))
+	}
+	l = len(m.Project)
+	if l > 0 {
+		n += 1 + l + sovTx(uint64(l))
+	}
+	if len(m.ProjectKeys) > 0 {
+		for _, e := range m.ProjectKeys {
+			l = e.Size()
+			n += 1 + l + sovTx(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *MsgDelKeysResponse) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	return n
+}
+
 func (m *MsgSetPolicy) Size() (n int) {
 	if m == nil {
 		return 0
@@ -1029,6 +1271,204 @@ func (m *MsgAddKeysResponse) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *MsgDelKeys) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTx
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: MsgDelKeys: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: MsgDelKeys: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Creator", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTx
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthTx
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthTx
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Creator = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Project", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTx
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthTx
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthTx
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Project = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 3:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ProjectKeys", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTx
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthTx
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthTx
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ProjectKeys = append(m.ProjectKeys, ProjectKey{})
+			if err := m.ProjectKeys[len(m.ProjectKeys)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTx(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthTx
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *MsgDelKeysResponse) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTx
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: MsgDelKeysResponse: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: MsgDelKeysResponse: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTx(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthTx
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *MsgSetPolicy) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0

From d56c633fb8134385b1d96359a2371d55041a814c Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sat, 3 Jun 2023 23:32:32 +0800
Subject: [PATCH 05/10] CNS-420: unit tests for DelKeysFromProject()

---
 x/projects/keeper/project_test.go | 172 +++++++++++++++++++++++++-----
 1 file changed, 144 insertions(+), 28 deletions(-)

diff --git a/x/projects/keeper/project_test.go b/x/projects/keeper/project_test.go
index c97f77c23e..20d75ce2dc 100644
--- a/x/projects/keeper/project_test.go
+++ b/x/projects/keeper/project_test.go
@@ -29,13 +29,15 @@ type testStruct struct {
 
 func newTestStruct(t *testing.T) *testStruct {
 	servers, keepers, _ctx := testkeeper.InitAllKeepers(t)
-	return &testStruct{
+	ts := &testStruct{
 		t:       t,
 		servers: servers,
 		keepers: keepers,
 		_ctx:    _ctx,
 		ctx:     sdk.UnwrapSDKContext(_ctx),
 	}
+	ts.AdvanceEpoch(1)
+	return ts
 }
 
 func (ts *testStruct) prepareData(numSub, numAdmin, numDevel int) {
@@ -46,12 +48,12 @@ func (ts *testStruct) prepareData(numSub, numAdmin, numDevel int) {
 		ts.accounts[k] = v
 	}
 	for i := 0; i < numAdmin; i++ {
-		k := "adm" + strconv.Itoa(i)
+		k := "adm" + strconv.Itoa(i+1)
 		v := common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
 		ts.accounts[k] = v
 	}
 	for i := 0; i < numDevel; i++ {
-		k := "dev" + strconv.Itoa(i)
+		k := "dev" + strconv.Itoa(i+1)
 		v := common.CreateNewAccount(ts._ctx, *ts.keepers, 10000).Addr.String()
 		ts.accounts[k] = v
 	}
@@ -131,15 +133,15 @@ func (ts *testStruct) AdvanceEpoch(count int) {
 
 func (ts *testStruct) isKeyInProject(index, key string, kind types.ProjectKey_Type) bool {
 	resp, err := ts.keepers.Projects.Info(ts._ctx, &types.QueryInfoRequest{Project: index})
-	require.Nil(ts.t, err, "project: " + index + ", key: " + key)
+	require.Nil(ts.t, err, "project: "+index+", key: "+key)
 	pk := resp.Project.GetKey(key)
 	return pk.IsType(kind)
 }
 
 func (ts *testStruct) addProjectKeys(index, creator string, projectKeys ...types.ProjectKey) error {
 	msg := types.MsgAddKeys{
-		Creator: creator,
-		Project: index,
+		Creator:     creator,
+		Project:     index,
 		ProjectKeys: projectKeys,
 	}
 	_, err := ts.servers.ProjectServer.AddKeys(ts._ctx, &msg)
@@ -148,8 +150,8 @@ func (ts *testStruct) addProjectKeys(index, creator string, projectKeys ...types
 
 func (ts *testStruct) delProjectKeys(index, creator string, projectKeys ...types.ProjectKey) error {
 	msg := types.MsgDelKeys{
-		Creator: creator,
-		Project: index,
+		Creator:     creator,
+		Project:     index,
 		ProjectKeys: projectKeys,
 	}
 	_, err := ts.servers.ProjectServer.DelKeys(ts._ctx, &msg)
@@ -181,11 +183,11 @@ func TestCreateProject(t *testing.T) {
 	ts := newTestStruct(t)
 	ts.prepareData(1, 0, 0) // 1 sub, 0 adm, 0 dev
 
-	projectData := ts.projects["pd1"]
+	projectData := ts.projects["pd2"]
 	plan := common.CreateMockPlan()
 
 	subAddr := ts.accounts["sub1"]
-	admAddr := ts.accounts["pd2_adm"]
+	admAddr := ts.accounts["pd2_both"]
 
 	err := ts.keepers.Projects.CreateProject(ts.ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
@@ -256,7 +258,7 @@ func TestCreateProject(t *testing.T) {
 
 	require.Equal(t, response2.Project, response1.Project)
 
-	_, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, response1.Project.Index, 0)
+	_, err = ts.keepers.Projects.GetProjectForBlock(ts.ctx, response1.Project.Index, ts.BlockHeight())
 	require.Nil(t, err)
 
 	// there should be one project key
@@ -270,11 +272,11 @@ func TestCreateProject(t *testing.T) {
 	require.True(t, response2.Project.ProjectKeys[0].IsType(types.ProjectKey_DEVELOPER))
 }
 
-func TestAddKeys(t *testing.T) {
+func TestAddDelKeys(t *testing.T) {
 	ts := newTestStruct(t)
 	ts.prepareData(1, 0, 2) // 1 sub, 0 adm, 2 dev
 
-	projectData := ts.projects["pd2"]
+	projectData := ts.projects["pd3"]
 	plan := common.CreateMockPlan()
 
 	subAddr := ts.accounts["sub1"]
@@ -318,14 +320,58 @@ func TestAddKeys(t *testing.T) {
 	err = ts.addProjectKeys(project.Index, admAddr, pk)
 	require.Nil(t, err)
 
+	// additions above will only take effect in next epoch
+	require.False(t, ts.isKeyInProject(project.Index, dev1Addr, types.ProjectKey_ADMIN))
+	require.False(t, ts.isKeyInProject(project.Index, dev2Addr, types.ProjectKey_DEVELOPER))
+	ts.AdvanceEpoch(1)
+	require.True(t, ts.isKeyInProject(project.Index, dev1Addr, types.ProjectKey_ADMIN))
+	require.True(t, ts.isKeyInProject(project.Index, dev2Addr, types.ProjectKey_DEVELOPER))
+
 	// new admin adding another developer
 	pk = types.ProjectDeveloperKey(dev3Addr)
 	err = ts.addProjectKeys(project.Index, dev1Addr, pk)
 	require.Nil(t, err)
 
+	// additions above will only take effect in next epoch
+	require.False(t, ts.isKeyInProject(project.Index, dev3Addr, types.ProjectKey_DEVELOPER))
+	ts.AdvanceEpoch(1)
+	require.True(t, ts.isKeyInProject(project.Index, dev3Addr, types.ProjectKey_DEVELOPER))
+
 	// fetch project with new developer
 	_, err = ts.keepers.Projects.Developer(ts._ctx, &types.QueryDeveloperRequest{Developer: dev3Addr})
 	require.Nil(t, err)
+
+	// developer delete admin
+	pk = types.ProjectAdminKey(admAddr)
+	err = ts.delProjectKeys(project.Index, dev2Addr, pk)
+	require.NotNil(t, err)
+
+	// developer delete developer
+	pk = types.ProjectDeveloperKey(dev3Addr)
+	err = ts.delProjectKeys(project.Index, dev2Addr, pk)
+	require.NotNil(t, err)
+
+	// new admin delete subscription owner (admin)
+	pk = types.ProjectAdminKey(subAddr)
+	err = ts.delProjectKeys(project.Index, admAddr, pk)
+	require.NotNil(t, err)
+
+	// subscription owner (admin) delete other admin
+	pk = types.ProjectAdminKey(admAddr)
+	err = ts.delProjectKeys(project.Index, subAddr, pk)
+	require.Nil(t, err)
+
+	// admin delete developer
+	pk = types.ProjectDeveloperKey(dev3Addr)
+	err = ts.delProjectKeys(project.Index, admAddr, pk)
+	require.Nil(t, err)
+
+	// deletion above will only take effect in next epoch
+	require.True(t, ts.isKeyInProject(project.Index, admAddr, types.ProjectKey_ADMIN))
+	require.True(t, ts.isKeyInProject(project.Index, dev3Addr, types.ProjectKey_DEVELOPER))
+	ts.AdvanceEpoch(1)
+	require.False(t, ts.isKeyInProject(project.Index, admAddr, types.ProjectKey_ADMIN))
+	require.False(t, ts.isKeyInProject(project.Index, dev3Addr, types.ProjectKey_DEVELOPER))
 }
 
 func TestAddAdminInTwoProjects(t *testing.T) {
@@ -528,23 +574,27 @@ func TestChargeComputeUnits(t *testing.T) {
 	subAddr := ts.accounts["pd1_adm"]
 	devAddr := ts.accounts["dev1"]
 
-	ts.AdvanceEpoch(1)
-	block1 := ts.BlockHeight()
-
 	err := ts.keepers.Projects.CreateProject(ts.ctx, subAddr, projectData, plan)
 	require.Nil(t, err)
 
 	ts.AdvanceEpoch(1)
-	block2 := ts.BlockHeight()
+	block1 := ts.BlockHeight()
 
 	projectID := types.ProjectIndex(subAddr, projectData.Name)
-	project, err := ts.keepers.Projects.GetProjectForBlock(ts.ctx, projectID, block2)
+	project, err := ts.keepers.Projects.GetProjectForBlock(ts.ctx, projectID, block1)
 	require.Nil(t, err)
 
 	// add developer key (created fixation)
 	err = ts.addProjectKeys(project.Index, subAddr, types.ProjectDeveloperKey(devAddr))
 	require.Nil(t, err)
 
+	// first epoch for the developer key addition to take place.
+
+	ts.AdvanceEpoch(1)
+	block2 := ts.BlockHeight()
+
+	// second epoch to move further, otherwise snapshot will affect the current new
+	// dev key instead of creating a separate fixation version.
 
 	ts.AdvanceEpoch(1)
 	block3 := ts.BlockHeight()
@@ -580,13 +630,18 @@ func TestChargeComputeUnits(t *testing.T) {
 	require.Equal(t, uint64(0), proj.UsedCu)
 }
 
-func TestAddDevKeysToSameProjectID(t *testing.T) {
+func TestAddDelKeysSameEpoch(t *testing.T) {
 	ts := newTestStruct(t)
 	ts.prepareData(2, 1, 5) // 2 sub, 1 adm, 5 dev
 
-	sub Addr := ts.accounts["sub1"]
+	sub1Addr := ts.accounts["sub1"]
+	sub2Addr := ts.accounts["sub2"]
+	adm1Addr := ts.accounts["adm1"]
 	dev1Addr := ts.accounts["dev1"]
 	dev2Addr := ts.accounts["dev2"]
+	dev3Addr := ts.accounts["dev3"]
+	dev4Addr := ts.accounts["dev4"]
+	dev5Addr := ts.accounts["dev5"]
 
 	plan := common.CreateMockPlan()
 
@@ -594,31 +649,91 @@ func TestAddDevKeysToSameProjectID(t *testing.T) {
 		Name:        "mockname1",
 		Description: "",
 		Enabled:     true,
-		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(subAddr)},
+		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(sub1Addr)},
 		Policy:      &plan.PlanPolicy,
 	}
-	err := keepers.Projects.CreateProject(_ctx, subAddr, projectData, plan)
+	err := ts.keepers.Projects.CreateProject(ts.ctx, sub1Addr, projectData1, plan)
 	require.Nil(t, err)
 
+	projectData2 := types.ProjectData{
+		Name:        "mockname2",
+		Description: "",
+		Enabled:     true,
+		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(sub2Addr)},
+		Policy:      &plan.PlanPolicy,
+	}
+	err = ts.keepers.Projects.CreateProject(ts.ctx, sub2Addr, projectData2, plan)
+	require.Nil(t, err)
 
-	projectID := types.ProjectIndex(sub1Addr, projectData1.Name)
+	ts.AdvanceEpoch(1)
 
-	ts.AdvanceBlock(1)
+	projectID1 := types.ProjectIndex(sub1Addr, projectData1.Name)
+	projectID2 := types.ProjectIndex(sub2Addr, projectData2.Name)
 
-	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID, sub1Addr,
+	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID1, sub1Addr,
 		[]types.ProjectKey{types.ProjectDeveloperKey(dev1Addr)})
 	require.Nil(t, err)
 
 	ts.AdvanceBlock(1)
 
-	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID, sub1Addr,
+	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID1, sub1Addr,
 		[]types.ProjectKey{types.ProjectDeveloperKey(dev2Addr)})
 	require.Nil(t, err)
 
+	require.False(t, ts.isKeyInProject(projectID1, dev1Addr, types.ProjectKey_DEVELOPER))
+	require.False(t, ts.isKeyInProject(projectID1, dev2Addr, types.ProjectKey_DEVELOPER))
+	ts.AdvanceEpoch(1)
+	require.True(t, ts.isKeyInProject(projectID1, dev1Addr, types.ProjectKey_DEVELOPER))
+	require.True(t, ts.isKeyInProject(projectID1, dev2Addr, types.ProjectKey_DEVELOPER))
 
-	proj, err := ts.keepers.Projects.GetProjectForDeveloper(ts.ctx, subAddr, ts.BlockHeight())
+	proj, err := ts.keepers.Projects.GetProjectForDeveloper(ts.ctx, sub1Addr, ts.BlockHeight())
 	require.Nil(t, err)
 	require.Equal(t, 3, len(proj.ProjectKeys))
+
+	// add twice - ok
+	err = ts.addProjectKeys(projectID1, sub1Addr, types.ProjectAdminKey(adm1Addr))
+	require.Nil(t, err)
+	err = ts.addProjectKeys(projectID1, sub1Addr, types.ProjectDeveloperKey(dev3Addr))
+	require.Nil(t, err)
+
+	ts.AdvanceEpoch(1)
+	require.True(t, ts.isKeyInProject(projectID1, adm1Addr, types.ProjectKey_ADMIN))
+	require.True(t, ts.isKeyInProject(projectID1, dev3Addr, types.ProjectKey_DEVELOPER))
+
+	// del twice - fail
+	err = ts.delProjectKeys(projectID1, sub1Addr, types.ProjectAdminKey(adm1Addr))
+	require.Nil(t, err)
+	err = ts.delProjectKeys(projectID1, sub1Addr, types.ProjectAdminKey(adm1Addr))
+	require.NotNil(t, err)
+	err = ts.delProjectKeys(projectID1, sub1Addr, types.ProjectDeveloperKey(dev3Addr))
+	require.Nil(t, err)
+	err = ts.delProjectKeys(projectID1, sub1Addr, types.ProjectDeveloperKey(dev3Addr))
+	require.NotNil(t, err)
+
+	ts.AdvanceEpoch(1)
+	require.False(t, ts.isKeyInProject(projectID1, adm1Addr, types.ProjectKey_ADMIN))
+	require.False(t, ts.isKeyInProject(projectID1, dev3Addr, types.ProjectKey_DEVELOPER))
+
+	// add, del in same epoch
+	err = ts.addProjectKeys(projectID2, sub2Addr, types.ProjectAdminKey(adm1Addr))
+	require.Nil(t, err)
+	err = ts.delProjectKeys(projectID2, sub2Addr, types.ProjectAdminKey(adm1Addr))
+	require.Nil(t, err)
+
+	err = ts.addProjectKeys(projectID2, sub2Addr, types.ProjectDeveloperKey(dev4Addr))
+	require.Nil(t, err)
+	err = ts.delProjectKeys(projectID2, sub2Addr, types.ProjectDeveloperKey(dev4Addr))
+	require.Nil(t, err)
+
+	ts.AdvanceEpoch(1)
+	require.False(t, ts.isKeyInProject(projectID2, adm1Addr, types.ProjectKey_ADMIN))
+	require.False(t, ts.isKeyInProject(projectID2, dev3Addr, types.ProjectKey_DEVELOPER))
+
+	// add dev to two projects in same epoch - latter fails
+	err = ts.addProjectKeys(projectID2, sub2Addr, types.ProjectDeveloperKey(dev5Addr))
+	require.Nil(t, err)
+	err = ts.addProjectKeys(projectID2, sub1Addr, types.ProjectDeveloperKey(dev5Addr))
+	require.NotNil(t, err)
 }
 
 func TestAddDevKeyToDifferentProjectsInSameBlock(t *testing.T) {
@@ -657,7 +772,7 @@ func TestAddDevKeyToDifferentProjectsInSameBlock(t *testing.T) {
 	err = ts.keepers.Projects.CreateProject(ts.ctx, sub2Addr, projectData2, plan)
 	require.Nil(t, err)
 
-	ts.AdvanceBlock(1)
+	ts.AdvanceEpoch(1)
 
 	err = ts.keepers.Projects.AddKeysToProject(ts.ctx, projectID1, sub1Addr,
 		[]types.ProjectKey{types.ProjectDeveloperKey(dev1Addr)})
@@ -667,6 +782,7 @@ func TestAddDevKeyToDifferentProjectsInSameBlock(t *testing.T) {
 		[]types.ProjectKey{types.ProjectDeveloperKey(dev1Addr)})
 	require.NotNil(t, err) // developer was already added to the first project
 
+	ts.AdvanceEpoch(1)
 
 	proj1, err := ts.keepers.Projects.GetProjectForDeveloper(ts.ctx, sub1Addr, ts.BlockHeight())
 	require.Nil(t, err)

From f3f2784f956ac6ba6a38eec453444a6bb4b248ed Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sun, 11 Jun 2023 15:21:46 +0800
Subject: [PATCH 06/10] CNS-420: [review] fixation improve comment

---
 common/fixation_entry.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/common/fixation_entry.go b/common/fixation_entry.go
index fc3d6fc404..329dbab526 100644
--- a/common/fixation_entry.go
+++ b/common/fixation_entry.go
@@ -164,7 +164,10 @@ func (fs *FixationStore) getEntryStore(ctx sdk.Context, index string) *prefix.St
 	return &store
 }
 
-func (fs *FixationStore) replaceTimer(ctx sdk.Context, prev, next types.Entry, block uint64, kind byte) {
+// transferTimer moves a timer (unexpired) from a previous entry to a new entry, with
+// the same expirty block. Useful, for example, when a newer entry takes responsibility
+// for a pending deletion from the previous owner.
+func (fs *FixationStore) transferTimer(ctx sdk.Context, prev, next types.Entry, block uint64, kind byte) {
 	key := encodeForTimer(prev.Index, prev.Block, kind)
 	fs.tstore.DelTimerByBlockHeight(ctx, block, key)
 
@@ -278,7 +281,7 @@ func (fs *FixationStore) AppendEntry(
 	}
 
 	if entry.HasDeleteAt() {
-		fs.replaceTimer(ctx, latestEntry, entry, entry.DeleteAt, timerDeleteEntry)
+		fs.transferTimer(ctx, latestEntry, entry, entry.DeleteAt, timerDeleteEntry)
 	}
 
 	fs.setEntry(ctx, entry)

From 227fa3838f2c9af52dd8f62a060a62b8572c1e3e Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sun, 11 Jun 2023 15:24:34 +0800
Subject: [PATCH 07/10] CNS-420: [review] add events for project key add/del

---
 x/projects/keeper/creation.go | 17 +++++++++++++++++
 x/projects/keeper/project.go  |  2 ++
 2 files changed, 19 insertions(+)

diff --git a/x/projects/keeper/creation.go b/x/projects/keeper/creation.go
index ff9f6a15f6..e44d4f2147 100644
--- a/x/projects/keeper/creation.go
+++ b/x/projects/keeper/creation.go
@@ -2,6 +2,7 @@ package keeper
 
 import (
 	"fmt"
+	"strconv"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
@@ -109,6 +110,14 @@ func (k Keeper) registerKey(ctx sdk.Context, key types.ProjectKey, project *type
 				)
 			}
 
+			logger := k.Logger(ctx)
+			details := map[string]string{
+				"project": project.GetIndex(),
+				"key":     key.Key,
+				"keytype": strconv.FormatInt(int64(key.Kinds), 10),
+			}
+			utils.LogLavaEvent(ctx, logger, types.AddProjectKeyEventName, details, "key added to project")
+
 			project.AppendKey(types.ProjectDeveloperKey(key.Key))
 		}
 	}
@@ -161,6 +170,14 @@ func (k Keeper) unregisterKey(ctx sdk.Context, key types.ProjectKey, project *ty
 		if !found {
 			panic("unregisterKey: developer key not found")
 		}
+
+		logger := k.Logger(ctx)
+		details := map[string]string{
+			"project": project.GetIndex(),
+			"key":     key.Key,
+			"keytype": strconv.FormatInt(int64(key.Kinds), 10),
+		}
+		utils.LogLavaEvent(ctx, logger, types.DelProjectKeyEventName, details, "key deleted from project")
 	}
 
 	return nil
diff --git a/x/projects/keeper/project.go b/x/projects/keeper/project.go
index 843d915cb9..9d28ceed25 100644
--- a/x/projects/keeper/project.go
+++ b/x/projects/keeper/project.go
@@ -69,6 +69,7 @@ func (k Keeper) AddKeysToProject(ctx sdk.Context, projectID string, adminKey str
 		return utils.LavaFormatWarning("failed to add keys",
 			fmt.Errorf("project not found"),
 			utils.Attribute{Key: "project", Value: projectID},
+			utils.Attribute{Key: "block", Value: ctxBlock},
 		)
 	}
 
@@ -85,6 +86,7 @@ func (k Keeper) AddKeysToProject(ctx sdk.Context, projectID string, adminKey str
 		return utils.LavaFormatError("failed to add keys",
 			fmt.Errorf("project not found (for next epoch)"),
 			utils.Attribute{Key: "project", Value: projectID},
+			utils.Attribute{Key: "block", Value: ctxBlock},
 		)
 	}
 

From 4ff9771b78466228f6c64c3cf28ff1ee3a3dd769 Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sun, 11 Jun 2023 15:26:12 +0800
Subject: [PATCH 08/10] CNS-420: [review] add project testing via server

---
 x/projects/keeper/project_test.go | 55 +++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/x/projects/keeper/project_test.go b/x/projects/keeper/project_test.go
index 20d75ce2dc..b617406a9c 100644
--- a/x/projects/keeper/project_test.go
+++ b/x/projects/keeper/project_test.go
@@ -272,6 +272,61 @@ func TestCreateProject(t *testing.T) {
 	require.True(t, response2.Project.ProjectKeys[0].IsType(types.ProjectKey_DEVELOPER))
 }
 
+func TestProjectsServerAPI(t *testing.T) {
+	ts := newTestStruct(t)
+	ts.prepareData(2, 1, 5) // 2 sub, 1 adm, 5 dev
+
+	sub1Acc := common.CreateNewAccount(ts._ctx, *ts.keepers, 20000)
+	sub1Addr := sub1Acc.Addr.String()
+
+	dev1Addr := ts.accounts["dev1"]
+	dev2Addr := ts.accounts["dev2"]
+
+	plan := common.CreateMockPlan()
+	ts.keepers.Plans.AddPlan(ts.ctx, plan)
+
+	common.BuySubscription(t, ts._ctx, *ts.keepers, *ts.servers, sub1Acc, plan.Index)
+
+	projectData := types.ProjectData{
+		Name:        "mockname2",
+		Description: "",
+		Enabled:     true,
+		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(dev1Addr)},
+		Policy:      &plan.PlanPolicy,
+	}
+
+	projectID := types.ProjectIndex(sub1Addr, projectData.Name)
+
+	msgAddProject := &subscriptiontypes.MsgAddProject{
+		Creator:     sub1Addr,
+		ProjectData: projectData,
+	}
+	_, err := ts.servers.SubscriptionServer.AddProject(ts._ctx, msgAddProject)
+	require.Nil(t, err)
+
+	ts.AdvanceEpoch(1)
+
+	msgAddKeys := types.MsgAddKeys{
+		Creator:     sub1Addr,
+		Project:     projectID,
+		ProjectKeys: []types.ProjectKey{types.ProjectDeveloperKey(dev2Addr)},
+	}
+	_, err = ts.servers.ProjectServer.AddKeys(ts._ctx, &msgAddKeys)
+	require.Nil(t, err)
+
+	ts.AdvanceBlock(1)
+
+	require.True(t, ts.isKeyInProject(projectID, dev1Addr, types.ProjectKey_DEVELOPER))
+	require.False(t, ts.isKeyInProject(projectID, dev2Addr, types.ProjectKey_DEVELOPER))
+	ts.AdvanceEpoch(1)
+	require.True(t, ts.isKeyInProject(projectID, dev2Addr, types.ProjectKey_DEVELOPER))
+
+	msgQueryDev := &types.QueryDeveloperRequest{Developer: sub1Addr}
+	res, err := ts.keepers.Projects.Developer(ts._ctx, msgQueryDev)
+	require.Nil(t, err)
+	require.Equal(t, 1, len(res.Project.ProjectKeys))
+}
+
 func TestAddDelKeys(t *testing.T) {
 	ts := newTestStruct(t)
 	ts.prepareData(1, 0, 2) // 1 sub, 0 adm, 2 dev

From 192c4879fee20a9712eda54e97150e19b28a3c54 Mon Sep 17 00:00:00 2001
From: Oren Laadan <orenl@lavanet.xyz>
Date: Sun, 11 Jun 2023 15:27:17 +0800
Subject: [PATCH 09/10] CNS-420: add e2e testing for add/del project keys

---
 cookbook/projects/example_project_keys.yml  |  1 -
 scripts/init_e2e.sh                         | 15 +++++++++++++++
 x/projects/types/types.go                   |  5 +++++
 x/subscription/client/cli/tx_add_project.go |  2 +-
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/cookbook/projects/example_project_keys.yml b/cookbook/projects/example_project_keys.yml
index b4c88db267..1cfe18bfb8 100644
--- a/cookbook/projects/example_project_keys.yml
+++ b/cookbook/projects/example_project_keys.yml
@@ -3,5 +3,4 @@ Project-Keys:
   - key: lava@1xtfqykth53pkt97v955h3lql8zkj2m4s4rq9cr
     Kinds: 3
   - key: lava@1r3ernqu6rzp95z92580wae7xpuqwmznk3eqd7w
-    types:
     Kinds: 1
diff --git a/scripts/init_e2e.sh b/scripts/init_e2e.sh
index aa10be435c..8e24ed8b80 100755
--- a/scripts/init_e2e.sh
+++ b/scripts/init_e2e.sh
@@ -17,6 +17,7 @@ lavad tx gov submit-proposal plans-add ./cookbook/plans/default.json,./cookbook/
 lavad tx gov vote 2 yes -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
 sleep 4
 
+# Plan removal (of one)
 lavad tx gov submit-proposal plans-del ./cookbook/plans/temporary-del.json -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
 lavad tx gov vote 3 yes -y --from alice --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
 
@@ -44,7 +45,21 @@ lavad tx pairing stake-provider "LAV1" $STAKE "127.0.0.1:2265,tendermintrpc,1 12
 lavad tx pairing stake-client "ETH1" $STAKE 1 -y --from user1 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
 lavad tx pairing stake-client "LAV1" $STAKE 1 -y --from user2 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
 
+# subscriptions and projects
 lavad tx subscription buy "DefaultPlan" -y --from user3 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
 
+user3addr=$(lavad keys show user3 -a)
+
+lavad tx subscription add-project "myproject" ./cookbook/projects/example_policy.yml -y --from user3 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE
+sleep_until_next_epoch
+
+lavad q subscription list-projects user3addr
+
+lavad tx project add-keys -y "$user3addr-myproject" --from user3 cookbook/projects/example_project_keys.yml --gas-prices=$GASPRICE
+sleep_until_next_epoch
+
+lavad tx project del-keys -y "$user3addr-myproject" --from user3 cookbook/projects/example_project_keys.yml --gas-prices=$GASPRICE
+sleep_until_next_epoch
+
 # we need to wait for the next epoch for the stake to take action.
 sleep_until_next_epoch
diff --git a/x/projects/types/types.go b/x/projects/types/types.go
index b66b3892c4..b533dc460c 100644
--- a/x/projects/types/types.go
+++ b/x/projects/types/types.go
@@ -12,3 +12,8 @@ const (
 	SET_ADMIN_POLICY        SetPolicyEnum = 1
 	SET_SUBSCRIPTION_POLICY SetPolicyEnum = 2
 )
+
+const (
+	AddProjectKeyEventName = "add_key_to_project_event"
+	DelProjectKeyEventName = "del_key_from_project_event"
+)
diff --git a/x/subscription/client/cli/tx_add_project.go b/x/subscription/client/cli/tx_add_project.go
index 682e976f9d..973f6633c9 100644
--- a/x/subscription/client/cli/tx_add_project.go
+++ b/x/subscription/client/cli/tx_add_project.go
@@ -33,7 +33,7 @@ func CmdAddProject() *cobra.Command {
 				  
 		optional flags: --policy-file <policy-file-path>, --project-keys-file <project-keys-file-path>, --disable
 				  
-		lavad tx subscription add-project [project-file-path] --from <subscription_consumer>`,
+		lavad tx subscription add-project --policy-file policy-file-path --from <subscription_consumer>`,
 		Args: cobra.RangeArgs(1, 2),
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			projectName := args[0]

From 137856faf2151f23aea61378ada6b50f19c9b77b Mon Sep 17 00:00:00 2001
From: Yarom Swisa <yarom@lavanet.xyz git config --global user.name Yarom>
Date: Sun, 11 Jun 2023 10:46:38 +0000
Subject: [PATCH 10/10] added upgrade handler v0.13.0

---
 app/app.go                     | 1 +
 app/upgrades/empty_upgrades.go | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/app/app.go b/app/app.go
index 238fe6b7b0..9efe48de41 100644
--- a/app/app.go
+++ b/app/app.go
@@ -161,6 +161,7 @@ var Upgrades = []upgrades.Upgrade{
 	upgrades.Upgrade_0_11_2,
 	upgrades.Upgrade_0_12_0,
 	upgrades.Upgrade_0_12_1,
+	upgrades.Upgrade_0_13_0,
 }
 
 // this line is used by starport scaffolding # stargate/wasm/app/enabledProposals
diff --git a/app/upgrades/empty_upgrades.go b/app/upgrades/empty_upgrades.go
index 13746c8623..934576ed64 100644
--- a/app/upgrades/empty_upgrades.go
+++ b/app/upgrades/empty_upgrades.go
@@ -177,3 +177,9 @@ var Upgrade_0_12_1 = Upgrade{
 	CreateUpgradeHandler: defaultUpgradeHandler,
 	StoreUpgrades:        store.StoreUpgrades{},
 }
+
+var Upgrade_0_13_0 = Upgrade{
+	UpgradeName:          "v0.13.0",
+	CreateUpgradeHandler: defaultUpgradeHandler,
+	StoreUpgrades:        store.StoreUpgrades{},
+}