Skip to content

Commit

Permalink
Replication: Add e2e tests for workloads-related rbac objects
Browse files Browse the repository at this point in the history 
Signed-off-by: David Festal <[email protected]>
  • Loading branch information
@davidfestal
davidfestal committed Mar 30, 2023
1 parent 6668a0c commit ae8c468
Showing 1 changed file with 193 additions and 1 deletion.
194 changes: 193 additions & 1 deletion test/e2e/reconciler/cache/replication_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/kcp-dev/logicalcluster/v3"
"github.com/stretchr/testify/require"

rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
Expand Down Expand Up @@ -64,6 +65,10 @@ var scenarios = []testScenario{
{"TestReplicateAPIResourceSchemaNegative", replicateAPIResourceSchemaNegativeScenario},
{"TestReplicateWorkspaceType", replicateWorkspaceTypeScenario},
{"TestReplicateWorkspaceTypeNegative", replicateWorkspaceTypeNegativeScenario},
{"TestReplicateWorkloadsClusterRole", replicateWorkloadsClusterRoleScenario},
{"TestReplicateWorkloadsClusterRoleNegative", replicateWorkloadsClusterRoleNegativeScenario},
{"TestReplicateWorkloadsClusterRoleBinding", replicateWorkloadsClusterRoleBindingScenario},
{"TestReplicateWorkloadsClusterRoleBindingNegative", replicateWorkloadsClusterRoleBindingNegativeScenario},
}

// disruptiveScenarios contains a list of scenarios that will be run in a private environment
Expand Down Expand Up @@ -678,14 +683,23 @@ func (b *replicateResourceScenario) verifyResourceReplicationHelper(ctx context.
}
unstructured.RemoveNestedField(originalResource.Object, "metadata", "resourceVersion")
unstructured.RemoveNestedField(cachedResource.Object, "metadata", "resourceVersion")

// TODO(davidfestal): find out why the generation is not equal, specially for rbacv1.
// Is it a characteristic of all built-in KCP resources (which are not backed by CRDs) ?
// Issue opened: https://github.com/kcp-dev/kcp/issues/2935
if b.gvr.Group == rbacv1.SchemeGroupVersion.Group {
unstructured.RemoveNestedField(originalResource.Object, "metadata", "generation")
unstructured.RemoveNestedField(cachedResource.Object, "metadata", "generation")
}

unstructured.RemoveNestedField(cachedResource.Object, "metadata", "annotations", genericapirequest.AnnotationKey)
if cachedStatus, ok := cachedResource.Object["status"]; ok && cachedStatus == nil || (cachedStatus != nil && len(cachedStatus.(map[string]interface{})) == 0) {
// TODO: worth investigating:
// for some reason cached resources have an empty status set whereas the original resources don't
unstructured.RemoveNestedField(cachedResource.Object, "status")
}
if diff := cmp.Diff(cachedResource.Object, originalResource.Object); len(diff) > 0 {
return false, fmt.Sprintf("replicated %s root|%s/%s is different from the original", b.gvr, cluster, cachedResourceMeta.GetName())
return false, fmt.Sprintf("replicated %s root|%s/%s is different from the original: %s", b.gvr, cluster, cachedResourceMeta.GetName(), diff)
}
return true, ""
}, wait.ForeverTestTimeout, 100*time.Millisecond)
Expand Down Expand Up @@ -732,3 +746,181 @@ func createCacheClientConfigForEnvironment(ctx context.Context, t *testing.T, kc
require.NoError(t, err)
return cacheServerRestConfig
}

// replicateWorkloadsClusterRoleScenario tests if a ClusterRole related to workloads API is propagated to the cache server.
// The test exercises creation, modification and removal of the Shard object.
func replicateWorkloadsClusterRoleScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
t.Helper()
replicateResource(ctx,
t,
server,
kcpShardClusterDynamicClient,
cacheKcpClusterDynamicClient,
"",
"ClusterRole",
rbacv1.SchemeGroupVersion.WithResource("clusterroles"),
&rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: withPseudoRandomSuffix("syncer"),
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"sync"},
APIGroups: []string{"workload.kcp.io"},
Resources: []string{"synctargets"},
ResourceNames: []string{"asynctarget"},
},
},
},
nil,
)
}

// replicateWorkloadsClusterRoleNegativeScenario checks if modified or even deleted cached ClusterRole (related to workloads API) will be reconciled to match the original object.
func replicateWorkloadsClusterRoleNegativeScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
t.Helper()
replicateResourceNegative(
ctx,
t,
server,
kcpShardClusterDynamicClient,
cacheKcpClusterDynamicClient,
"",
"ClusterRole",
rbacv1.SchemeGroupVersion.WithResource("clusterroles"),
&rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: withPseudoRandomSuffix("syncer"),
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"sync"},
APIGroups: []string{"workload.kcp.io"},
Resources: []string{"synctargets"},
ResourceNames: []string{"asynctarget"},
},
},
},
nil,
)
}

// replicateWorkloadsClusterRoleBindingScenario tests if a ClusterRoleBinding related to workloads API is propagated to the cache server.
// The test exercises creation, modification and removal of the Shard object.
func replicateWorkloadsClusterRoleBindingScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
t.Helper()

clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: withPseudoRandomSuffix("syncer"),
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"sync"},
APIGroups: []string{"workload.kcp.io"},
Resources: []string{"synctargets"},
ResourceNames: []string{"asynctarget"},
},
},
}

orgPath, _ := framework.NewOrganizationFixture(t, server)
_, ws := framework.NewWorkspaceFixture(t, server, orgPath, framework.WithRootShard())
clusterName := logicalcluster.Name(ws.Spec.Cluster)

t.Logf("Create additional ClusterRole %s on the root shard for replication", clusterRole.Name)
clusterRoleGVR := rbacv1.SchemeGroupVersion.WithResource("clusterroles")
clusterRoleUnstr, err := toUnstructured(clusterRole, "ClusterRole", clusterRoleGVR)
require.NoError(t, err)
_, err = kcpShardClusterDynamicClient.Resource(clusterRoleGVR).Cluster(clusterName.Path()).Create(ctx, clusterRoleUnstr, metav1.CreateOptions{})
require.NoError(t, err)

replicateResource(ctx,
t,
server,
kcpShardClusterDynamicClient,
cacheKcpClusterDynamicClient,
clusterName,
"ClusterRoleBinding",
rbacv1.SchemeGroupVersion.WithResource("clusterrolebindings"),
&rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: withPseudoRandomSuffix("syncer"),
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.SchemeGroupVersion.Group,
Kind: "ClusterRole",
Name: clusterRole.Name,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
APIGroup: "",
Name: "kcp-syncer-0000",
Namespace: "kcp-syncer-namespace",
},
},
},
nil,
)
}

// replicateWorkloadsClusterRoleNegativeScenario checks if modified or even deleted cached ClusterRole (related to workloads API) will be reconciled to match the original object.
func replicateWorkloadsClusterRoleBindingNegativeScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
t.Helper()

clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: withPseudoRandomSuffix("syncer"),
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"sync"},
APIGroups: []string{"workload.kcp.io"},
Resources: []string{"synctargets"},
ResourceNames: []string{"asynctarget"},
},
},
}

orgPath, _ := framework.NewOrganizationFixture(t, server)
_, ws := framework.NewWorkspaceFixture(t, server, orgPath, framework.WithRootShard())
clusterName := logicalcluster.Name(ws.Spec.Cluster)

t.Logf("Create additional ClusterRole %s on the root shard for replication", clusterRole.Name)
clusterRoleGVR := rbacv1.SchemeGroupVersion.WithResource("clusterroles")
clusterRoleUnstr, err := toUnstructured(clusterRole, "ClusterRole", clusterRoleGVR)
require.NoError(t, err)
_, err = kcpShardClusterDynamicClient.Resource(clusterRoleGVR).Cluster(clusterName.Path()).Create(ctx, clusterRoleUnstr, metav1.CreateOptions{})
require.NoError(t, err)

replicateResourceNegative(
ctx,
t,
server,
kcpShardClusterDynamicClient,
cacheKcpClusterDynamicClient,
clusterName,
"ClusterRoleBinding",
rbacv1.SchemeGroupVersion.WithResource("clusterrolebindings"),
&rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: withPseudoRandomSuffix("syncer"),
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.SchemeGroupVersion.Group,
Kind: "ClusterRole",
Name: clusterRole.Name,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
APIGroup: "",
Name: "kcp-syncer-0000",
Namespace: "kcp-syncer-namespace",
},
},
},
nil,
)
}

0 comments on commit ae8c468

Please sign in to comment.