check license from nsxt side

While init, it will check the license from nsxt side. If CONTAINER
license is disable, it will reboot. If DFW license is disable,
security policy will be only response for DELETE operation.
It will run a routine to check license periodically. If there is
no DFW license, it will check license more frequently

SecurityPolicy controller will check if error is invalid license error.

Test Done:
no CONTAINER license
1. if no CONTAINER license, nsx-operator should reset
CONTAINER license enable, DFW disable
1. nsx-operator could bootup
2. security policy failed to create or update
3. security policy could be deleted
CONTAINER license enable, DFW enable -> CONTAINER/DFW disable
1. nsx-operator restart due to DFW changed
no DFW license, but nsx-operator try to create security policy
1. nsx-operator restart due to invalid license error

cmd/main.go

@@ -42,6 +42,7 @@ import (
 	subnetservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet"
 	subnetportservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 var (
@@ -168,6 +169,8 @@ func main() {
 		NSXConfig: cf,
 	}
 
+	checkLicense(nsxClient, cf.LicenseValidationInterval, config.LicenseTimeoutNoDFW)
+
 	var vpcService *vpc.VPCService
 
 	if cf.CoeConfig.EnableVPCNetwork && commonService.NSXClient.NSXCheckVersion(nsx.VPC) {
@@ -269,3 +272,29 @@ func updateHealthMetricsPeriodically(nsxClient *nsx.Client) {
 		}
 	}
 }
+
+func checkLicense(nsxClient *nsx.Client, interval int, licenseTimeoutNoDFW int) {
+	err := nsxClient.ValidateLicense(true)
+	if err != nil {
+		os.Exit(1)
+	}
+	// if there is no dfw license enabled, check license more frequently
+	if !util.IsLicensed(util.FeatureDFW) {
+		if interval > licenseTimeoutNoDFW {
+			interval = licenseTimeoutNoDFW
+		}
+	}
+	go updateLicensePeriodically(nsxClient, time.Duration(interval)*time.Second)
+}
+
+func updateLicensePeriodically(nsxClient *nsx.Client, interval time.Duration) {
+	for {
+		select {
+		case <-time.After(interval):
+		}
+		err := nsxClient.ValidateLicense(false)
+		if err != nil {
+			os.Exit(1)
+		}
+	}
+}

pkg/config/config.go

@@ -22,6 +22,10 @@ import (
 const (
 	nsxOperatorDefaultConf = "/etc/nsx-operator/nsxop.ini"
 	vcHostCACertPath       = "/etc/vmware/wcp/tls/vmca.pem"
+	// LicenseTimeout is the timeout for checking license status
+	LicenseTimeout = 7200
+	// LicenseTimeoutNoDFW is the timeout for checking license status while no DFW license enabled
+	LicenseTimeoutNoDFW = 1800
 )
 
 var (
@@ -88,22 +92,23 @@ type CoeConfig struct {
 }
 
 type NsxConfig struct {
-	NsxApiUser           string   `ini:"nsx_api_user"`
-	NsxApiPassword       string   `ini:"nsx_api_password"`
-	NsxApiCertFile       string   `ini:"nsx_api_cert_file"`
-	NsxApiPrivateKeyFile string   `ini:"nsx_api_private_key_file"`
-	NsxApiManagers       []string `ini:"nsx_api_managers"`
-	CaFile               []string `ini:"ca_file"`
-	Thumbprint           []string `ini:"thumbprint"`
-	Insecure             bool     `ini:"insecure"`
-	SingleTierSrTopology bool     `ini:"single_tier_sr_topology"`
-	EnforcementPoint     string   `ini:"enforcement_point"`
-	DefaultProject       string   `ini:"default_project"`
-	ExternalIPv4Blocks   []string `ini:"external_ipv4_blocks"`
-	DefaultSubnetSize    int      `ini:"default_subnet_size"`
-	DefaultTimeout       int      `ini:"default_timeout"`
-	EnvoyHost            string   `ini:"envoy_host"`
-	EnvoyPort            int      `ini:"envoy_port"`
+	NsxApiUser                string   `ini:"nsx_api_user"`
+	NsxApiPassword            string   `ini:"nsx_api_password"`
+	NsxApiCertFile            string   `ini:"nsx_api_cert_file"`
+	NsxApiPrivateKeyFile      string   `ini:"nsx_api_private_key_file"`
+	NsxApiManagers            []string `ini:"nsx_api_managers"`
+	CaFile                    []string `ini:"ca_file"`
+	Thumbprint                []string `ini:"thumbprint"`
+	Insecure                  bool     `ini:"insecure"`
+	SingleTierSrTopology      bool     `ini:"single_tier_sr_topology"`
+	EnforcementPoint          string   `ini:"enforcement_point"`
+	DefaultProject            string   `ini:"default_project"`
+	ExternalIPv4Blocks        []string `ini:"external_ipv4_blocks"`
+	DefaultSubnetSize         int      `ini:"default_subnet_size"`
+	DefaultTimeout            int      `ini:"default_timeout"`
+	EnvoyHost                 string   `ini:"envoy_host"`
+	EnvoyPort                 int      `ini:"envoy_port"`
+	LicenseValidationInterval int      `ini:"license_validation_interval"`
 }
 
 type K8sConfig struct {
@@ -365,6 +370,10 @@ func (nsxConfig *NsxConfig) validate(enableVPC bool) error {
 	if err := nsxConfig.validateCert(); err != nil {
 		return err
 	}
+	if nsxConfig.LicenseValidationInterval == 0 {
+		// set default value of LicenseValidationInterval
+		nsxConfig.LicenseValidationInterval = LicenseTimeout
+	}
 	return nil
 }
 

pkg/controllers/securitypolicy/securitypolicy_controller.go

@@ -129,6 +129,26 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				updateFail(r, &ctx, obj, &err)
 				return ResultNormal, nil
 			}
+			// check if invalid license
+			apiErr, _ := nsxutil.DumpAPIError(err)
+			if apiErr != nil {
+				invalidLicense := false
+				errorMessage := ""
+				for _, apiErrItem := range apiErr.RelatedErrors {
+					if *apiErrItem.ErrorCode == nsxutil.InvalidLicenseErrorCode {
+						invalidLicense = true
+						errorMessage = *apiErrItem.ErrorMessage
+					}
+				}
+				if *apiErr.ErrorCode == nsxutil.InvalidLicenseErrorCode {
+					invalidLicense = true
+					errorMessage = *apiErr.ErrorMessage
+				}
+				if invalidLicense {
+					log.Error(err, "Invalid license, nsx-operator will restart", "error message", errorMessage)
+					os.Exit(1)
+				}
+			}
 			log.Error(err, "create or update failed, would retry exponentially", "securitypolicy", req.NamespacedName)
 			updateFail(r, &ctx, obj, &err)
 			return ResultRequeue, err

pkg/nsx/client.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 const (
@@ -265,3 +266,31 @@ func (client *Client) NSXCheckVersion(feature int) bool {
 func (client *Client) FeatureEnabled(feature int) bool {
 	return client.NSXVerChecker.featureSupported[feature] == true
 }
+
+// ValidateLicense validates NSX license. init is used to indicate whether nsx-operator is init or not
+// if not init, nsx-operator will check if license has been updated.
+// once license updated, operator will restart
+// if FeatureContainer license is false, operatore will restart
+func (client *Client) ValidateLicense(init bool) error {
+	log.Info("Checking NSX license")
+	oldContainerLicense := util.IsLicensed(util.FeatureContainer)
+	oldDfwLicense := util.IsLicensed(util.FeatureDFW)
+	err := client.NSXChecker.cluster.FetchLicense()
+	if err != nil {
+		return err
+	}
+	if !util.IsLicensed(util.FeatureContainer) {
+		err = errors.New("NSX license check failed")
+		log.Error(err, "container license is not supported")
+		return err
+	}
+	if !init {
+		newContainerLicense := util.IsLicensed(util.FeatureContainer)
+		newDfwLicense := util.IsLicensed(util.FeatureDFW)
+		if newContainerLicense != oldContainerLicense || newDfwLicense != oldDfwLicense {
+			log.Info("license updated, reset", "container license new value", newContainerLicense, "DFW license new value", newDfwLicense, "container license old value", oldContainerLicense, "DFW license old value", oldDfwLicense)
+			return errors.New("license updated")
+		}
+	}
+	return nil
+}

pkg/nsx/cluster.go

@@ -5,9 +5,11 @@ package nsx
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"math/big"
 	"net"
 	"net/http"
 	"os"
@@ -39,6 +41,12 @@ const (
 const (
 	EnvoyUrlWithCert       = "http://%s:%d/external-cert/http1/%s"
 	EnvoyUrlWithThumbprint = "http://%s:%d/external-tp/http1/%s/%s"
+	LicenseAPI             = "api/v1/licenses/licensed-features"
+)
+
+const (
+	maxNSXGetRetries = 10
+	NSXGetDelay      = 2 * time.Second
 )
 
 // Cluster consists of endpoint and provides http.Client used to send http requests.
@@ -460,3 +468,58 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool {
 	}
 	return false
 }
+
+func (cluster *Cluster) createHttpRequest(api string, ep *Endpoint) (*http.Request, error) {
+	return http.NewRequest("GET", fmt.Sprintf("%s://%s/%s", ep.Scheme(), ep.Host(), api), nil)
+}
+
+func (cluster *Cluster) getLicenseFromNsx() (*util.NsxLicense, error) {
+	ep := cluster.endpoints[0]
+	req, err := cluster.createHttpRequest(LicenseAPI, ep)
+	if err != nil {
+		log.Error(err, "failed to create http request")
+		return nil, err
+	}
+	err = ep.UpdateHttpRequestAuth(req)
+	if err != nil {
+		log.Error(err, "keep alive update auth error")
+		return nil, err
+	}
+
+	resp, err := ep.noBalancerClient.Do(req)
+	if err != nil {
+		log.Error(err, "failed to get nsx license")
+		return nil, err
+	}
+	nsxLicense := &util.NsxLicense{}
+	err, _ = util.HandleHTTPResponse(resp, nsxLicense, true)
+	return nsxLicense, err
+}
+
+func (cluster *Cluster) getLicenseWithRetries(delay time.Duration, maxRetry int) (*util.NsxLicense, error) {
+	var err error
+	for i := 0; i < maxRetry; i++ {
+		nsxLicense, err := cluster.getLicenseFromNsx()
+		if err != nil {
+			log.Error(err, "failed to get nsx license")
+			rand, err := rand.Int(rand.Reader, big.NewInt(1000))
+			if err != nil {
+				log.Error(err, "failed to generate random number")
+				return nil, err
+			}
+			time.Sleep(delay + time.Duration(rand.Int64())*time.Millisecond)
+		} else {
+			return nsxLicense, nil
+		}
+	}
+	return nil, err
+}
+
+func (cluster *Cluster) FetchLicense() error {
+	nsxLicense, err := cluster.getLicenseWithRetries(NSXGetDelay, maxNSXGetRetries)
+	if err != nil {
+		return err
+	}
+	util.UpdateFeatureLicense(nsxLicense)
+	return nil
+}

pkg/nsx/cluster_test.go

@@ -4,8 +4,11 @@
 package nsx
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -17,6 +20,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 func TestNewCluster(t *testing.T) {
@@ -344,3 +348,70 @@ func TestCluster_CreateServerUrl(t *testing.T) {
 		})
 	}
 }
+
+func TestGetLicenseFromNsx(t *testing.T) {
+	address := address{
+		host:   "1.2.3.4",
+		scheme: "https",
+	}
+	// Success case
+	cluster := &Cluster{endpoints: []*Endpoint{{
+		provider: &address,
+	}}}
+
+	// Request creation failure
+	patch := gomonkey.ApplyFunc(http.NewRequest,
+		func(method, url string, body io.Reader) (*http.Request, error) {
+			return nil, errors.New("request error")
+		})
+	license, err := cluster.getLicenseFromNsx()
+	assert.Error(t, err)
+	assert.Nil(t, license)
+	patch.Reset()
+
+	// HTTP error
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			return nil, errors.New("http error")
+		})
+
+	license, err = cluster.getLicenseFromNsx()
+	assert.Error(t, err)
+	assert.Nil(t, license)
+	patch.Reset()
+
+	// normal case
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			res := &nsxutil.NsxLicense{
+				Results: []struct {
+					FeatureName string `json:"feature_name"`
+					IsLicensed  bool   `json:"is_licensed"`
+				}{{
+					FeatureName: "CONTAINER",
+					IsLicensed:  true,
+				},
+					{
+						FeatureName: "DFW",
+						IsLicensed:  true,
+					},
+				},
+				ResultCount: 2,
+			}
+
+			jsonBytes, _ := json.Marshal(res)
+
+			return &http.Response{
+				StatusCode: 200,
+				Body:       io.NopCloser(bytes.NewReader(jsonBytes)),
+				Header: http.Header{
+					"Content-Type": []string{"application/json"},
+				},
+				Request: req,
+			}, nil
+		})
+	defer patch.Reset()
+	_, err = cluster.getLicenseFromNsx()
+	assert.Nil(t, err)
+
+}

pkg/nsx/services/securitypolicy/firewall.go

@@ -137,6 +137,10 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi
 }
 
 func (service *SecurityPolicyService) CreateOrUpdateSecurityPolicy(obj interface{}) error {
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		log.Info("no DFW license, skip creating SecurityPolicy.")
+		return nsxutil.RestrictionError{Desc: "no DFW license"}
+	}
 	var err error
 	switch obj.(type) {
 	case *networkingv1.NetworkPolicy:

pkg/nsx/services/vpc/vpc.go

@@ -625,6 +625,9 @@ func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace strin
 	if !enableAviAllowRule {
 		return nil
 	}
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		return nil
+	}
 	vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path)
 	if err != nil {
 		log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path)

pkg/nsx/services/vpc/vpc_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 var (
@@ -471,6 +472,7 @@ func TestCreateOrUpdateAVIRule(t *testing.T) {
 	sp := model.SecurityPolicy{
 		Path: &sppath1,
 	}
+	util.UpdateLicense(util.FeatureDFW, true)
 
 	// security policy not found
 	spClient.SP = sp

pkg/nsx/util/errors.go

@@ -7,6 +7,10 @@ import (
 	"fmt"
 )
 
+const (
+	InvalidLicenseErrorCode = 505
+)
+
 type NsxError interface {
 	setDetail(detail *ErrorDetail)
 	Error() string