diff --git a/pkg/clean/clean.go b/pkg/clean/clean.go index a2b3b4409..403f99a7a 100644 --- a/pkg/clean/clean.go +++ b/pkg/clean/clean.go @@ -58,7 +58,6 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, er NSXClient: nsxClient, NSXConfig: cf, } - vpcService, vpcErr := vpc.InitializeVPC(commonService) commonctl.ServiceMediator.VPCService = vpcService diff --git a/pkg/controllers/vpc/vpc_controller.go b/pkg/controllers/vpc/vpc_controller.go index b8e5d0afc..d3e987cf0 100644 --- a/pkg/controllers/vpc/vpc_controller.go +++ b/pkg/controllers/vpc/vpc_controller.go @@ -70,6 +70,12 @@ func (r *VPCReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R updateFail(r.Service.NSXConfig, &ctx, obj, &err, r.Client) return common.ResultRequeueAfter10sec, err } + err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace) + if err != nil { + log.Error(err, "operate failed, would retry exponentially", "VPC", req.NamespacedName) + updateFail(r.Service.NSXConfig, &ctx, obj, &err, r.Client) + return common.ResultRequeueAfter10sec, err + } snatIP, path, cidr := "", "", "" // currently, auto snat is not exposed, and use default value True diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 205058dc1..a010729a6 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -23,6 +23,7 @@ import ( "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/infra/realized_state" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs" nat "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/nat" + vpc_sp "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/security_policies" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets/ip_pools" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets/ports" @@ -39,10 +40,11 @@ const ( ServiceAccountRestore ServiceAccountCertRotation StaticRoute + VpcAviRule AllFeatures ) -var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_ACCOUNT", "NSX_SERVICE_ACCOUNT_RESTORE", "NSX_SERVICE_ACCOUNT_CERT_ROTATION", "STATIC_ROUTE"} +var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_ACCOUNT", "NSX_SERVICE_ACCOUNT_RESTORE", "NSX_SERVICE_ACCOUNT_CERT_ROTATION", "STATIC_ROUTE", "VPC_AVI_RULE"} type Client struct { NsxConfig *config.NSXOperatorConfig @@ -63,6 +65,10 @@ type Client struct { PrincipalIdentitiesClient trust_management.PrincipalIdentitiesClient WithCertificateClient principal_identities.WithCertificateClient + // for AVI security policy rule + VPCSecurityClient vpcs.SecurityPoliciesClient + VPCRuleClient vpc_sp.RulesClient + OrgRootClient nsx_policy.OrgRootClient ProjectInfraClient projects.InfraClient VPCClient projects.VpcsClient @@ -94,12 +100,8 @@ type NSXHealthChecker struct { } type NSXVersionChecker struct { - cluster *Cluster - securityPolicySupported bool - nsxServiceAccountSupported bool - nsxServiceAccountRestoreSupported bool - vpcSupported bool - featureSupported [AllFeatures]bool + cluster *Cluster + featureSupported [AllFeatures]bool } func (ck *NSXHealthChecker) CheckNSXHealth(req *http.Request) error { @@ -158,6 +160,9 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) + vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) + vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) + nsxChecker := &NSXHealthChecker{ cluster: cluster, } @@ -193,6 +198,8 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { PortClient: portClient, PortStateClient: portStateClient, SubnetStatusClient: subnetStatusClient, + VPCSecurityClient: vpcSecurityClient, + VPCRuleClient: vpcRuleClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, @@ -215,6 +222,10 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { err := errors.New("NSXServiceAccountRestore feature support check failed") log.Error(err, "initial NSX version check for NSXServiceAccountRestore got error") } + if !nsxClient.NSXCheckVersion(VpcAviRule) { + err := errors.New("VpcAviRule feature support check failed") + log.Error(err, "initial NSX version check for VpcAviRule got error") + } if !nsxClient.NSXCheckVersion(ServiceAccountCertRotation) { err := errors.New("ServiceAccountCertRotation feature support check failed") log.Error(err, "initial NSX version check for ServiceAccountCertRotation got error") @@ -247,3 +258,7 @@ func (client *Client) NSXCheckVersion(feature int) bool { client.NSXVerChecker.featureSupported[feature] = true return true } + +func (client *Client) FeatureEnabled(feature int) bool { + return client.NSXVerChecker.featureSupported[feature] == true +} diff --git a/pkg/nsx/cluster.go b/pkg/nsx/cluster.go index edabe7d83..cd8e69f5d 100644 --- a/pkg/nsx/cluster.go +++ b/pkg/nsx/cluster.go @@ -331,6 +331,9 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool { case ServiceAccountCertRotation: minVersion = nsx413Version validFeature = true + case VpcAviRule: + minVersion = nsx411Version + validFeature = true } if validFeature { diff --git a/pkg/nsx/services/common/store.go b/pkg/nsx/services/common/store.go index 29a15385f..1e41147aa 100644 --- a/pkg/nsx/services/common/store.go +++ b/pkg/nsx/services/common/store.go @@ -132,40 +132,13 @@ func (service *Service) InitializeVPCResourceStore(wg *sync.WaitGroup, fatalErro service.InitializeCommonStore(wg, fatalErrors, org, project, resourceTypeValue, tags, store) } -// InitializeCommonStore is the common method used by InitializeResourceStore and InitializeVPCResourceStore -func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors chan error, org string, project string, resourceTypeValue string, tags []model.Tag, store Store) { - defer wg.Done() - - tagScopeClusterKey := strings.Replace(TagScopeCluster, "/", "\\/", -1) - tagScopeClusterValue := strings.Replace(service.NSXClient.NsxConfig.Cluster, ":", "\\:", -1) - tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) - - for _, tag := range tags { - tagKey := strings.Replace(*tag.Scope, "/", "\\/", -1) - tagParam += fmt.Sprintf(" AND tags.scope:%s ", tagKey) - if tag.Tag != nil { - tagValue := strings.Replace(*tag.Tag, ":", "\\:", -1) - tagParam += fmt.Sprintf(" AND tags.tag:%s ", tagValue) - } - } - - resourceParam := fmt.Sprintf("%s:%s", ResourceType, resourceTypeValue) - queryParam := resourceParam + " AND " + tagParam - - if org != "" || project != "" { - // QueryClient.List() will escape the path, "path:" then will be "path%25%3A" instead of "path:3A", - //"path%25%3A" would fail to get response. Hack it here. - path := "\\/orgs\\/" + org + "\\/projects\\/" + project + "\\/*" - pathUnescape, _ := url.PathUnescape("path%3A") - queryParam += " AND " + pathUnescape + path - } - queryParam += " AND marked_for_delete:false" +type Filter func(interface{}) *data.StructValue +func (service *Service) SearchResource(resourceTypeValue string, queryParam string, store Store, filter Filter) (uint64, error) { var cursor *string = nil count := uint64(0) for { var err error - var results []*data.StructValue var resultCount *int64 if store.IsPolicyAPI() { @@ -181,18 +154,22 @@ func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors ch resultCount = response.ResultCount err = searchEerr } + err = TransError(err) if _, ok := err.(nsxutil.PageMaxError); ok == true { DecrementPageSize(Int64(PageSize)) continue } if err != nil { - fatalErrors <- err + return count, err } for _, entity := range results { + if filter != nil { + entity = filter(entity) + } err = store.TransResourceToStore(entity) if err != nil { - fatalErrors <- err + return count, err } count++ } @@ -204,5 +181,44 @@ func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors ch break } } + return count, nil +} + +// PopulateResourcetoStore is the method used by populating resources created not by nsx-operator +func (service *Service) PopulateResourcetoStore(wg *sync.WaitGroup, fatalErrors chan error, resourceTypeValue string, queryParam string, store Store, filter Filter) { + defer wg.Done() + count, err := service.SearchResource(resourceTypeValue, queryParam, store, filter) + if err != nil { + fatalErrors <- err + } log.Info("initialized store", "resourceType", resourceTypeValue, "count", count) } + +// InitializeCommonStore is the common method used by InitializeResourceStore and InitializeVPCResourceStore +func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors chan error, org string, project string, resourceTypeValue string, tags []model.Tag, store Store) { + tagScopeClusterKey := strings.Replace(TagScopeCluster, "/", "\\/", -1) + tagScopeClusterValue := strings.Replace(service.NSXClient.NsxConfig.Cluster, ":", "\\:", -1) + tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) + + for _, tag := range tags { + tagKey := strings.Replace(*tag.Scope, "/", "\\/", -1) + tagParam += fmt.Sprintf(" AND tags.scope:%s ", tagKey) + if tag.Tag != nil { + tagValue := strings.Replace(*tag.Tag, ":", "\\:", -1) + tagParam += fmt.Sprintf(" AND tags.tag:%s ", tagValue) + } + } + + resourceParam := fmt.Sprintf("%s:%s", ResourceType, resourceTypeValue) + queryParam := resourceParam + " AND " + tagParam + + if org != "" || project != "" { + // QueryClient.List() will escape the path, "path:" then will be "path%25%3A" instead of "path:3A", + //"path%25%3A" would fail to get response. Hack it here. + path := "\\/orgs\\/" + org + "\\/projects\\/" + project + "\\/*" + pathUnescape, _ := url.PathUnescape("path%3A") + queryParam += " AND " + pathUnescape + path + } + queryParam += " AND marked_for_delete:false" + service.PopulateResourcetoStore(wg, fatalErrors, resourceTypeValue, queryParam, store, nil) +} diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 0db0319ad..ee675f923 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -62,6 +62,7 @@ const ( TagValueGroupScope string = "scope" TagValueGroupSrc string = "source" TagValueGroupDst string = "destination" + TagValueGroupAvi string = "avi" AnnotationVPCNetworkConfig string = "nsx.vmware.com/vpc_network_config" AnnotationVPCName string = "nsx.vmware.com/vpc_name" AnnotationPodMAC string = "nsx.vmware.com/mac" diff --git a/pkg/nsx/services/vpc/store.go b/pkg/nsx/services/vpc/store.go index 37cdc443f..1efd6ddf7 100644 --- a/pkg/nsx/services/vpc/store.go +++ b/pkg/nsx/services/vpc/store.go @@ -147,3 +147,86 @@ func (is *IPBlockStore) GetByIndex(index string, value string) *model.IpAddressB block := indexResults[0].((model.IpAddressBlock)) return &block } + +// keyFuncAVI is used to get the key of a AVI rule related resource +func keyFuncAVI(obj interface{}) (string, error) { + switch v := obj.(type) { + case model.Rule: + return *v.Path, nil + case model.SecurityPolicy: + return *v.Path, nil + case model.Group: + return *v.Path, nil + case model.IpAddressBlock: + return *v.Path, nil + default: + return "", errors.New("keyFunc doesn't support unknown type") + } +} + +// AviRuleStore is a store for saving AVI related Rules in VPCs +type AviRuleStore struct { + common.ResourceStore +} + +func (ruleStore *AviRuleStore) Apply(i interface{}) error { + return nil +} +func (ruleStore *AviRuleStore) GetByKey(key string) *model.Rule { + obj := ruleStore.ResourceStore.GetByKey(key) + if obj != nil { + rule := obj.(model.Rule) + return &rule + } + return nil +} + +// PubIPblockStore is a store to query external ip blocks cidr +type PubIPblockStore struct { + common.ResourceStore +} + +func (ipBlockStore *PubIPblockStore) Apply(i interface{}) error { + return nil +} +func (ipBlockStore *PubIPblockStore) GetByKey(key string) *model.IpAddressBlock { + obj := ipBlockStore.ResourceStore.GetByKey(key) + if obj != nil { + ipblock := obj.(model.IpAddressBlock) + return &ipblock + } + return nil +} + +type AviGroupStore struct { + common.ResourceStore +} + +func (groupStore *AviGroupStore) Apply(i interface{}) error { + return nil +} +func (groupStore *AviGroupStore) GetByKey(key string) *model.Group { + obj := groupStore.ResourceStore.GetByKey(key) + if obj != nil { + group := obj.(model.Group) + return &group + } + return nil +} + +type AviSecurityPolicyStore struct { + common.ResourceStore +} + +func (securityPolicyStore *AviSecurityPolicyStore) Apply(i interface{}) error { + return nil +} + +func (securityPolicyStore *AviSecurityPolicyStore) GetByKey(key string) *model.SecurityPolicy { + obj := securityPolicyStore.ResourceStore.GetByKey(key) + if obj != nil { + sp := obj.(model.SecurityPolicy) + return &sp + } + return nil +} diff --git a/pkg/nsx/services/vpc/store_test.go b/pkg/nsx/services/vpc/store_test.go index be84dfe37..9afb00d24 100644 --- a/pkg/nsx/services/vpc/store_test.go +++ b/pkg/nsx/services/vpc/store_test.go @@ -253,3 +253,102 @@ func TestVPCStore_CRUDResource_List(t *testing.T) { }) } } + +func TestRuleStore_GetByKey(t *testing.T) { + vpcRuleCacheIndexer := cache.NewIndexer(keyFuncAVI, nil) + resourceStore := common.ResourceStore{ + Indexer: vpcRuleCacheIndexer, + BindingType: model.RuleBindingType(), + } + ruleStore := &AviRuleStore{ResourceStore: resourceStore} + service := &VPCService{ + Service: common.Service{NSXClient: nil}, + } + service.RuleStore = ruleStore + + path1 := "/org/default/project/project_1/vpcs/vpc1/security-policies/default-section/rules/rule1" + path2 := "/org/default/project/project_1/vpcs/vpc2/security-policies/default-section/rules/rule1" + rule1 := model.Rule{ + Path: &path1, + } + rule2 := model.Rule{ + Path: &path2, + } + ruleStore.Add(rule1) + + rule := ruleStore.GetByKey(path1) + assert.Equal(t, rule.Path, rule1.Path) + + rule = ruleStore.GetByKey(path2) + assert.True(t, rule == nil) + + ruleStore.Add(rule2) + rule = ruleStore.GetByKey(path2) + assert.Equal(t, rule.Path, rule2.Path) +} + +func TestGroupStore_GetByKey(t *testing.T) { + groupCacheIndexer := cache.NewIndexer(keyFuncAVI, nil) + resourceStore := common.ResourceStore{ + Indexer: groupCacheIndexer, + BindingType: model.GroupBindingType(), + } + groupStore := &AviGroupStore{ResourceStore: resourceStore} + service := &VPCService{ + Service: common.Service{NSXClient: nil}, + } + service.GroupStore = groupStore + + path1 := "/org/default/project/project_1/vpcs/vpc1/groups/group1" + path2 := "/org/default/project/project_1/vpcs/vpc2/groups/group2" + group1 := model.Group{ + Path: &path1, + } + group2 := model.Group{ + Path: &path2, + } + groupStore.Add(group1) + + group := groupStore.GetByKey(path1) + assert.Equal(t, group.Path, group1.Path) + + group = groupStore.GetByKey(path2) + assert.True(t, group == nil) + + groupStore.Add(group2) + group = groupStore.GetByKey(path2) + assert.Equal(t, group.Path, group2.Path) +} + +func TestSecurityPolicyStore_GetByKey(t *testing.T) { + spCacheIndexer := cache.NewIndexer(keyFuncAVI, nil) + resourceStore := common.ResourceStore{ + Indexer: spCacheIndexer, + BindingType: model.SecurityPolicyBindingType(), + } + spStore := &AviSecurityPolicyStore{ResourceStore: resourceStore} + service := &VPCService{ + Service: common.Service{NSXClient: nil}, + } + service.SecurityPolicyStore = spStore + + path1 := "/org/default/project/project_1/vpcs/vpc1/security-policies/default-section" + path2 := "/org/default/project/project_1/vpcs/vpc2/security-policies/default-section" + sp1 := model.SecurityPolicy{ + Path: &path1, + } + sp2 := model.SecurityPolicy{ + Path: &path2, + } + spStore.Add(sp1) + + sp := spStore.GetByKey(path1) + assert.Equal(t, sp.Path, sp1.Path) + + sp = spStore.GetByKey(path2) + assert.True(t, sp == nil) + + spStore.Add(sp2) + sp = spStore.GetByKey(path2) + assert.Equal(t, sp.Path, sp2.Path) +} diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 9ee63f275..2e9648985 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -4,22 +4,35 @@ import ( "context" "errors" "fmt" + "math" "net" "strings" "sync" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/logger" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate" ) +const ( + AviSEIngressAllowRuleId = "avi-se-ingress-allow-rule" + VPCAviSEGroupId = "avi-se-vms" + VpcDefaultSecurityPolicyId = "default-layer3-section" + GroupKey = "/orgs/%s/projects/%s/vpcs/%s/groups/%s" + SecurityPolicyKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s" + RuleKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s" +) + var ( log = logger.Log ctx = context.Background() @@ -39,12 +52,20 @@ var ( resourceType = "resource_type" EnforceRevisionCheckParam = false MarkedForDelete = true + enableAviAllowRule = false ) type VPCService struct { common.Service VpcStore *VPCStore IpblockStore *IPBlockStore + AVIAllowRule +} +type AVIAllowRule struct { + GroupStore *AviGroupStore + RuleStore *AviRuleStore + SecurityPolicyStore *AviSecurityPolicyStore + PubIpblockStore *PubIPblockStore } func (s *VPCService) RegisterVPCNetworkConfig(ncCRName string, info VPCNetworkConfigInfo) { @@ -106,10 +127,15 @@ func InitializeVPC(service common.Service) (*VPCService, error) { wgDone := make(chan bool) fatalErrors := make(chan error) - wg.Add(2) - VPCService := &VPCService{Service: service} - + enableAviAllowRule = service.NSXClient.FeatureEnabled(nsx.VpcAviRule) + if enableAviAllowRule { + log.Info("support avi allow rule") + wg.Add(5) + } else { + log.Info("disable avi allow rule") + wg.Add(2) + } VPCService.VpcStore = &VPCStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeVPCCRUID: indexFunc}), BindingType: model.VpcBindingType(), @@ -126,6 +152,31 @@ func InitializeVPC(service common.Service) (*VPCService, error) { go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeVpc, nil, VPCService.VpcStore) go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeIPBlock, nil, VPCService.IpblockStore) + //initalize avi rule related store + if enableAviAllowRule { + VPCService.RuleStore = &AviRuleStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFuncAVI, nil), + BindingType: model.RuleBindingType(), + }} + VPCService.GroupStore = &AviGroupStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFuncAVI, nil), + BindingType: model.GroupBindingType(), + }} + VPCService.SecurityPolicyStore = &AviSecurityPolicyStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFuncAVI, nil), + BindingType: model.SecurityPolicyBindingType(), + }} + VPCService.PubIpblockStore = &PubIPblockStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFuncAVI, nil), + BindingType: model.IpAddressBlockBindingType(), + }} + go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeGroup, nil, VPCService.GroupStore) + go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeRule, nil, VPCService.RuleStore) + + query := fmt.Sprintf("%s:%s AND visibility:EXTERNAL", common.ResourceType, common.ResourceTypeIPBlock) + go VPCService.PopulateResourcetoStore(&wg, fatalErrors, common.ResourceTypeIPBlock, query, VPCService.PubIpblockStore, nil) + } + go func() { wg.Wait() close(wgDone) @@ -361,19 +412,19 @@ func (s *VPCService) CreateorUpdateVPC(obj *v1alpha1.VPC) (*model.Vpc, *VPCNetwo } // read corresponding vpc network config from store - nc_name, err := s.getNetworkconfigNameFromNS(obj.Namespace) + ncName, err := s.getNetworkconfigNameFromNS(obj.Namespace) if err != nil { log.Error(err, "failed to get network config name for VPC when creating NSX VPC", "VPC", obj.Name) return nil, nil, err } - nc, _exist := s.GetVPCNetworkConfig(nc_name) + nc, _exist := s.GetVPCNetworkConfig(ncName) if !_exist { - message := fmt.Sprintf("failed to read network config %s when creating NSX VPC", nc_name) + message := fmt.Sprintf("failed to read network config %s when creating NSX VPC", ncName) log.Info(message) return nil, nil, errors.New(message) } - log.Info("read network config from store", "NetworkConfig", nc_name) + log.Info("read network config from store", "NetworkConfig", ncName) paths, err := s.CreatOrUpdatePrivateIPBlock(obj, nc) if err != nil { @@ -470,3 +521,232 @@ func (s *VPCService) Cleanup() error { return nil } + +func (service *VPCService) needUpdateRule(rule *model.Rule, externalCIDRs []string) bool { + des := rule.DestinationGroups + currentDesSet := sets.Set[string]{} + for _, group := range des { + currentDesSet.Insert(group) + } + if len(externalCIDRs) != len(currentDesSet) { + return true + } + for _, cidr := range externalCIDRs { + if !currentDesSet.Has(cidr) { + return true + } + } + return false +} + +func (service *VPCService) getIpblockCidr(blocks []string) (result []string, err error) { + for _, cidr := range blocks { + ipblock := service.PubIpblockStore.GetByKey(cidr) + if ipblock == nil { + // in case VPC using the new ipblock, search the ipblock from nsxt + // return error, and retry next time when the ipblock is synced into store + err = errors.New("ipblock not found") + log.Error(err, "failed to get public ipblock", "path", cidr) + query := fmt.Sprintf("%s:%s AND visibility:EXTERNAL", common.ResourceType, common.ResourceTypeIPBlock) + count, searcherr := service.SearchResource(common.ResourceTypeIPBlock, query, service.PubIpblockStore, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query public ipblock", "query", query) + } else { + log.V(1).Info("query public ipblock", "count", count) + } + return + } else { + result = append(result, *ipblock.Cidr) + } + } + return +} + +func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace string) error { + if !enableAviAllowRule { + return nil + } + vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path) + orgId := vpcInfo.OrgID + projectId := vpcInfo.ProjectID + ruleId := AviSEIngressAllowRuleId + groupId := VPCAviSEGroupId + spId := VpcDefaultSecurityPolicyId + + if !service.checkAVISecurityPolicyExist(orgId, projectId, *vpc.Id, spId) { + return errors.New("avi security policy not found") + } + allowrule, err := service.getAVIAllowRule(orgId, projectId, *vpc.Id, spId, ruleId) + externalCIDRs, err := service.getIpblockCidr(vpc.ExternalIpv4Blocks) + if err != nil { + return err + } else { + log.Info("avi rule get external cidr", "cidr", externalCIDRs) + } + + if allowrule != nil { + if !service.needUpdateRule(allowrule, externalCIDRs) { + log.Info("avi rule is not changed, skip updating avi rulee") + return nil + } else { + log.Info("avi rule changed", "previous", allowrule.DestinationGroups, "current", externalCIDRs) + } + } + + group, err := service.getorCreateAVIGroup(orgId, projectId, *vpc.Id, groupId) + if err != nil { + log.Error(err, "failed to get avi group", "group", groupId) + return err + } + + newrule, err := service.buildAVIAllowRule(vpc, externalCIDRs, *group.Path, ruleId, projectId) + log.Info("creating avi rule", "rule", newrule) + if err != nil { + log.Error(err, "failed to build avi rule", "rule", newrule) + return err + } + + err = service.NSXClient.VPCRuleClient.Patch(orgId, projectId, *vpc.Id, spId, *newrule.Id, *newrule) + if err != nil { + log.Error(err, "failed to create avi rule", "rule", newrule) + return err + } + nsxrule, err := service.NSXClient.VPCRuleClient.Get(orgId, projectId, *vpc.Id, spId, *newrule.Id) + if err != nil { + log.Error(err, "failed to get avi rule", "rule", nsxrule) + return err + } + service.RuleStore.Add(nsxrule) + log.Info("created avi rule successfully") + return nil +} + +func (service *VPCService) getorCreateAVIGroup(orgId string, projectId string, vpcId string, groupId string) (*model.Group, error) { + group, err := service.getAVIGroup(orgId, projectId, vpcId, groupId) + if err != nil { + log.Info("create avi group", "group", groupId) + group, err = service.createAVIGroup(orgId, projectId, vpcId, groupId) + if err != nil { + log.Error(err, "failed to create avi group", "group", groupId) + return group, err + } + service.GroupStore.Add(group) + } + return group, err +} + +func (service *VPCService) buildAVIGroupTag(vpcId string) []model.Tag { + return []model.Tag{ + { + Scope: common.String(common.TagScopeCluster), + Tag: common.String(service.NSXConfig.Cluster), + }, + { + Scope: common.String(common.TagScopeVersion), + Tag: common.String(strings.Join(common.TagValueVersion, ".")), + }, + { + Scope: common.String(common.TagScopeVPCCRUID), + Tag: common.String(vpcId), + }, + { + Scope: common.String(common.TagScopeGroupType), + Tag: common.String(common.TagValueGroupAvi), + }, + } +} + +func (service *VPCService) createAVIGroup(orgId string, projectId string, vpcId string, groupId string) (*model.Group, error) { + group := model.Group{} + group.Tags = service.buildAVIGroupTag(vpcId) + expression := service.buildExpression("Condition", "VpcSubnet", "AVI_SUBNET_LB|", "Tag", "EQUALS", "EQUALS") + group.Expression = []*data.StructValue{expression} + group.DisplayName = common.String(groupId) + + err := service.NSXClient.VpcGroupClient.Patch(orgId, projectId, vpcId, groupId, group) + if err != nil { + return &group, err + } + nsxgroup, err := service.NSXClient.VpcGroupClient.Get(orgId, projectId, vpcId, groupId) + return &nsxgroup, err +} + +func (service *VPCService) buildExpression(resource_type, member_type, value, key, operator, scope_op string) *data.StructValue { + return data.NewStructValue( + "", + map[string]data.DataValue{ + "resource_type": data.NewStringValue(resource_type), + "member_type": data.NewStringValue(member_type), + "value": data.NewStringValue(value), + "key": data.NewStringValue(key), + "operator": data.NewStringValue(operator), + "scope_operator": data.NewStringValue(scope_op), + }, + ) +} + +func (service *VPCService) buildAVIAllowRule(obj *model.Vpc, externalCIDRs []string, groupId, ruleId, projectId string) (*model.Rule, error) { + rule := &model.Rule{} + rule.Action = common.String(model.Rule_ACTION_ALLOW) + rule.Direction = common.String(model.Rule_DIRECTION_IN_OUT) + rule.Scope = append(rule.Scope, groupId) + rule.SequenceNumber = common.Int64(math.MaxInt32 - 1) + rule.DestinationGroups = externalCIDRs + rule.SourceGroups = append(rule.SourceGroups, "Any") + name := fmt.Sprintf("PROJECT-%s-VPC-%s-%s", projectId, *obj.Id, ruleId) + rule.DisplayName = common.String(name) + rule.Id = common.String(ruleId) + rule.Services = []string{"ANY"} + rule.IsDefault = common.Bool(true) + tags := []model.Tag{ + { + Scope: common.String(common.TagScopeCluster), + Tag: common.String(service.NSXConfig.Cluster), + }, + { + Scope: common.String(common.TagScopeVersion), + Tag: common.String(strings.Join(common.TagValueVersion, ".")), + }, + } + rule.Tags = tags + return rule, nil +} + +func (service *VPCService) getAVIAllowRule(orgId string, projectId string, vpcId string, spId string, ruleId string) (*model.Rule, error) { + key := fmt.Sprintf(RuleKey, orgId, projectId, vpcId, spId, ruleId) + rule := service.RuleStore.GetByKey(key) + if rule == nil { + log.Info("avi rule not found", "key", key) + return nil, errors.New("avi rule not found") + } + return rule, nil +} + +func (service *VPCService) getAVIGroup(orgId string, projectId string, vpcId string, groupId string) (*model.Group, error) { + key := fmt.Sprintf(GroupKey, orgId, projectId, vpcId, groupId) + group := service.GroupStore.GetByKey(key) + var err error + if group == nil { + log.Info("avi se group not found", "key", key) + err = errors.New("avi se group not found") + } + return group, err +} + +// checkAVISecurityPolicyExist returns true if security policy for that VPC already exists +// this security policy created by NSXT once VPC created +// if not found, wait until it created +func (service *VPCService) checkAVISecurityPolicyExist(orgId string, projectId string, vpcId string, spId string) bool { + key := fmt.Sprintf(SecurityPolicyKey, orgId, projectId, vpcId, spId) + sp := service.SecurityPolicyStore.GetByKey(key) + if sp != nil { + return true + } + nsxtsp, err := service.NSXClient.VPCSecurityClient.Get(orgId, projectId, vpcId, spId) + if err != nil { + log.Error(err, "failed to get avi security policy", "key", key) + return false + } + service.SecurityPolicyStore.Add(nsxtsp) + return true +} diff --git a/pkg/nsx/services/vpc/vpc_test.go b/pkg/nsx/services/vpc/vpc_test.go index 832d81a10..c7623b8a7 100644 --- a/pkg/nsx/services/vpc/vpc_test.go +++ b/pkg/nsx/services/vpc/vpc_test.go @@ -1,12 +1,18 @@ package vpc import ( + "errors" + "fmt" + "reflect" "testing" + "github.com/agiledragon/gomonkey/v2" "github.com/stretchr/testify/assert" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" "k8s.io/client-go/tools/cache" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" ) @@ -126,6 +132,7 @@ func TestVPC_GetVPCsByNamespace(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { vpcStore.Apply(&vpc1) + vpcStore.GetByKey(vpcID1) vpcStore.Apply(&vpc2) got := vpcStore.List() if len(got) != 2 { @@ -142,3 +149,225 @@ func TestVPC_GetVPCsByNamespace(t *testing.T) { }) } } + +type MockSecurityPoliciesClient struct { + SP model.SecurityPolicy + Err error +} + +func (spClient *MockSecurityPoliciesClient) Delete(orgIdParam string, projectIdParam string, vpcIdParam string, groupIdParam string) error { + return spClient.Err +} + +func (spClient *MockSecurityPoliciesClient) Get(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string) (model.SecurityPolicy, error) { + return spClient.SP, spClient.Err +} + +func (spClient *MockSecurityPoliciesClient) List(orgIdParam string, projectIdParam string, vpcIdParam string, cursorParam *string, includeMarkForDeleteObjectsParam *bool, includeRuleCountParam *bool, includedFieldsParam *string, pageSizeParam *int64, sortAscendingParam *bool, sortByParam *string) (model.SecurityPolicyListResult, error) { + return model.SecurityPolicyListResult{}, spClient.Err +} + +func (spClient *MockSecurityPoliciesClient) Patch(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, securityPolicyParam model.SecurityPolicy) error { + return spClient.Err +} +func (spClient *MockSecurityPoliciesClient) Update(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, securityPolicyParam model.SecurityPolicy) (model.SecurityPolicy, error) { + return spClient.SP, spClient.Err +} +func (spClient *MockSecurityPoliciesClient) Revise(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, securityPolicyParam model.SecurityPolicy, anchorPathParam *string, operationParam *string) (model.SecurityPolicy, error) { + return model.SecurityPolicy{}, spClient.Err +} + +type MockGroupClient struct { + Group model.Group + Err error +} + +func (groupClient *MockGroupClient) Delete(orgIdParam string, projectIdParam string, vpcIdParam string, groupIdParam string) error { + return groupClient.Err +} + +func (groupClient *MockGroupClient) Get(orgIdParam string, projectIdParam string, vpcIdParam string, groupIdParam string) (model.Group, error) { + return groupClient.Group, groupClient.Err +} + +func (groupClient *MockGroupClient) List(orgIdParam string, projectIdParam string, vpcIdParam string, cursorParam *string, includeMarkForDeleteObjectsParam *bool, includedFieldsParam *string, memberTypesParam *string, pageSizeParam *int64, sortAscendingParam *bool, sortByParam *string) (model.GroupListResult, error) { + return model.GroupListResult{}, groupClient.Err +} + +func (groupClient *MockGroupClient) Patch(orgIdParam string, projectIdParam string, vpcIdParam string, groupIdParam string, groupParam model.Group) error { + return groupClient.Err +} +func (groupClient *MockGroupClient) Update(orgIdParam string, projectIdParam string, vpcIdParam string, groupIdParam string, groupParam model.Group) (model.Group, error) { + return groupClient.Group, groupClient.Err +} + +type MockRuleClient struct { + Rule model.Rule + Err error +} + +func (ruleClient *MockRuleClient) Delete(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, ruleIdParam string) error { + return ruleClient.Err +} + +func (ruleClient *MockRuleClient) Get(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, ruleIdParam string) (model.Rule, error) { + return ruleClient.Rule, ruleClient.Err +} +func (ruleClient *MockRuleClient) List(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, cursorParam *string, includeMarkForDeleteObjectsParam *bool, includedFieldsParam *string, pageSizeParam *int64, sortAscendingParam *bool, sortByParam *string) (model.RuleListResult, error) { + return model.RuleListResult{}, ruleClient.Err +} + +func (ruleClient *MockRuleClient) Patch(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, ruleIdParam string, ruleParam model.Rule) error { + return ruleClient.Err +} + +func (ruleClient *MockRuleClient) Update(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, ruleIdParam string, ruleParam model.Rule) (model.Rule, error) { + return ruleClient.Rule, ruleClient.Err +} + +func (ruleClient *MockRuleClient) Revise(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, ruleIdParam string, ruleParam model.Rule, anchorPathParam *string, operationParam *string) (model.Rule, error) { + return model.Rule{}, ruleClient.Err +} + +func TestVPC_CreateOrUpdateAVIRule(t *testing.T) { + aviRuleCacheIndexer := cache.NewIndexer(keyFuncAVI, nil) + resourceStore := common.ResourceStore{ + Indexer: aviRuleCacheIndexer, + BindingType: model.VpcBindingType(), + } + ruleStore := &AviRuleStore{ResourceStore: resourceStore} + resourceStore1 := common.ResourceStore{ + Indexer: aviRuleCacheIndexer, + BindingType: model.GroupBindingType(), + } + groupStore := &AviGroupStore{ResourceStore: resourceStore1} + resourceStore2 := common.ResourceStore{ + Indexer: aviRuleCacheIndexer, + BindingType: model.SecurityPolicyBindingType(), + } + spStore := &AviSecurityPolicyStore{ResourceStore: resourceStore2} + + service := &VPCService{ + Service: common.Service{NSXClient: nil}, + } + + service.RuleStore = ruleStore + service.GroupStore = groupStore + service.SecurityPolicyStore = spStore + + ns1 := "test-ns-1" + tag1 := []model.Tag{ + { + Scope: &tagScopeCluster, + Tag: &cluster, + }, + { + Scope: &tagScopeNamespace, + Tag: &ns1, + }, + { + Scope: &tagScopeVPCCRName, + Tag: &tagValueVPCCRName, + }, + { + Scope: &tagScopeVPCCRUID, + Tag: &tagValueVPCCRUID, + }, + } + path1 := "/orgs/default/projects/project_1/vpcs/vpc1" + vpc1 := model.Vpc{ + Path: &path1, + DisplayName: &vpcName1, + Id: &vpcID1, + Tags: tag1, + IpAddressType: &IPv4Type, + PrivateIpv4Blocks: []string{"1.1.1.0/24"}, + ExternalIpv4Blocks: []string{"2.2.2.0/24"}, + } + + // feature not supported + enableAviAllowRule = false + err := service.CreateOrUpdateAVIRule(&vpc1, ns1) + assert.Equal(t, err, nil) + + // enable feature + enableAviAllowRule = true + spClient := MockSecurityPoliciesClient{} + + service.NSXClient = &nsx.Client{} + service.NSXClient.VPCSecurityClient = &spClient + service.NSXConfig = &config.NSXOperatorConfig{} + service.NSXConfig.CoeConfig = &config.CoeConfig{} + service.NSXConfig.Cluster = "k8scl_one" + sppath1 := "/orgs/default/projects/project_1/vpcs/vpc1/security-policies/sp1" + sp := model.SecurityPolicy{ + Path: &sppath1, + } + + // security policy not found + spClient.SP = sp + notFound := errors.New("avi security policy not found") + spClient.Err = notFound + err = service.CreateOrUpdateAVIRule(&vpc1, ns1) + assert.Equal(t, err, notFound) + + // security policy found, get rule, failed to get external CIDR + rulepath1 := fmt.Sprintf("/orgs/default/projects/project_1/vpcs/ns-vpc-uid-1/security-policies/default-layer3-section/rules/%s", AviSEIngressAllowRuleId) + rule := model.Rule{ + Path: &rulepath1, + DestinationGroups: []string{"2.2.2.0/24"}, + } + ruleStore.Add(rule) + spClient.Err = nil + resulterr := errors.New("get external ipblock failed") + patch := gomonkey.ApplyPrivateMethod(reflect.TypeOf(service), "getIpblockCidr", func(_ *VPCService, cidr []string) ([]string, error) { + return []string{}, resulterr + }) + err = service.CreateOrUpdateAVIRule(&vpc1, ns1) + patch.Reset() + assert.Equal(t, err, resulterr) + + // security policy found, get rule, get external CIDR which matched + spClient.Err = nil + resulterr = errors.New("get external ipblock failed") + patch = gomonkey.ApplyPrivateMethod(reflect.TypeOf(service), "getIpblockCidr", func(_ *VPCService, cidr []string) ([]string, error) { + return []string{"2.2.2.0/24"}, nil + }) + err = service.CreateOrUpdateAVIRule(&vpc1, ns1) + patch.Reset() + assert.Equal(t, err, nil) + + // security policy found, get external CIDR, create group failed + patch = gomonkey.ApplyPrivateMethod(reflect.TypeOf(service), "getIpblockCidr", func(_ *VPCService, cidr []string) ([]string, error) { + return []string{"192.168.0.0/16"}, nil + }) + defer patch.Reset() + groupClient := MockGroupClient{Err: nil} + service.NSXClient.VpcGroupClient = &groupClient + grouppath1 := "/orgs/default/projects/project_1/vpcs/vpc1/groups/group1" + group := model.Group{ + Path: &grouppath1, + } + groupClient.Group = group + groupClient.Err = errors.New("create avi group error") + service.NSXConfig = &config.NSXOperatorConfig{} + service.NSXConfig.CoeConfig = &config.CoeConfig{} + service.NSXConfig.Cluster = "k8scl-one" + err = service.CreateOrUpdateAVIRule(&vpc1, ns1) + assert.Equal(t, err, groupClient.Err) + + // security policy found, get external CIDR, create group, create rule failed + groupClient.Err = nil + ruleClient := MockRuleClient{} + service.NSXClient.VPCRuleClient = &ruleClient + + ruleClient.Rule = rule + ruleClient.Err = errors.New("create avi rule error") + err = service.CreateOrUpdateAVIRule(&vpc1, ns1) + assert.Equal(t, err, ruleClient.Err) + + // security policy found, get external CIDR, create group, create rule + ruleClient.Err = nil + err = service.CreateOrUpdateAVIRule(&vpc1, ns1) + assert.Equal(t, err, nil) +}