Merge pull request vmware-tanzu#496 from TaoZou1/license

check license from nsxt side

cmd/main.go

@@ -43,6 +43,7 @@ import (
 	subnetservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet"
 	subnetportservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 var (
@@ -169,6 +170,8 @@ func main() {
 		NSXConfig: cf,
 	}
 
+	checkLicense(nsxClient, cf.LicenseValidationInterval)
+
 	var vpcService *vpc.VPCService
 
 	if cf.CoeConfig.EnableVPCNetwork {
@@ -276,3 +279,32 @@ func updateHealthMetricsPeriodically(nsxClient *nsx.Client) {
 		}
 	}
 }
+
+func checkLicense(nsxClient *nsx.Client, interval int) {
+	err := nsxClient.ValidateLicense(true)
+	if err != nil {
+		os.Exit(1)
+	}
+	// if there is no dfw license enabled, check license more frequently
+	// if customer set it in config, use it, else use licenseTimeoutNoDFW
+	if interval == 0 {
+		if !util.IsLicensed(util.FeatureDFW) {
+			interval = config.LicenseIntervalForDFW
+		} else {
+			interval = config.LicenseInterval
+		}
+	}
+	go updateLicensePeriodically(nsxClient, time.Duration(interval)*time.Second)
+}
+
+func updateLicensePeriodically(nsxClient *nsx.Client, interval time.Duration) {
+	for {
+		select {
+		case <-time.After(interval):
+		}
+		err := nsxClient.ValidateLicense(false)
+		if err != nil {
+			os.Exit(1)
+		}
+	}
+}

pkg/config/config.go

@@ -22,6 +22,10 @@ import (
 const (
 	nsxOperatorDefaultConf = "/etc/nsx-operator/nsxop.ini"
 	vcHostCACertPath       = "/etc/vmware/wcp/tls/vmca.pem"
+	// LicenseInterval is the timeout for checking license status
+	LicenseInterval = 7200
+	// LicenseIntervalForDFW is the timeout for checking license status while no DFW license enabled
+	LicenseIntervalForDFW = 1800
 )
 
 var (
@@ -88,22 +92,23 @@ type CoeConfig struct {
 }
 
 type NsxConfig struct {
-	NsxApiUser           string   `ini:"nsx_api_user"`
-	NsxApiPassword       string   `ini:"nsx_api_password"`
-	NsxApiCertFile       string   `ini:"nsx_api_cert_file"`
-	NsxApiPrivateKeyFile string   `ini:"nsx_api_private_key_file"`
-	NsxApiManagers       []string `ini:"nsx_api_managers"`
-	CaFile               []string `ini:"ca_file"`
-	Thumbprint           []string `ini:"thumbprint"`
-	Insecure             bool     `ini:"insecure"`
-	SingleTierSrTopology bool     `ini:"single_tier_sr_topology"`
-	EnforcementPoint     string   `ini:"enforcement_point"`
-	DefaultProject       string   `ini:"default_project"`
-	ExternalIPv4Blocks   []string `ini:"external_ipv4_blocks"`
-	DefaultSubnetSize    int      `ini:"default_subnet_size"`
-	DefaultTimeout       int      `ini:"default_timeout"`
-	EnvoyHost            string   `ini:"envoy_host"`
-	EnvoyPort            int      `ini:"envoy_port"`
+	NsxApiUser                string   `ini:"nsx_api_user"`
+	NsxApiPassword            string   `ini:"nsx_api_password"`
+	NsxApiCertFile            string   `ini:"nsx_api_cert_file"`
+	NsxApiPrivateKeyFile      string   `ini:"nsx_api_private_key_file"`
+	NsxApiManagers            []string `ini:"nsx_api_managers"`
+	CaFile                    []string `ini:"ca_file"`
+	Thumbprint                []string `ini:"thumbprint"`
+	Insecure                  bool     `ini:"insecure"`
+	SingleTierSrTopology      bool     `ini:"single_tier_sr_topology"`
+	EnforcementPoint          string   `ini:"enforcement_point"`
+	DefaultProject            string   `ini:"default_project"`
+	ExternalIPv4Blocks        []string `ini:"external_ipv4_blocks"`
+	DefaultSubnetSize         int      `ini:"default_subnet_size"`
+	DefaultTimeout            int      `ini:"default_timeout"`
+	EnvoyHost                 string   `ini:"envoy_host"`
+	EnvoyPort                 int      `ini:"envoy_port"`
+	LicenseValidationInterval int      `ini:"license_validation_interval"`
 }
 
 type K8sConfig struct {

pkg/controllers/securitypolicy/securitypolicy_controller.go

@@ -128,6 +128,26 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				updateFail(r, &ctx, obj, &err)
 				return ResultNormal, nil
 			}
+			// check if invalid license
+			apiErr, _ := nsxutil.DumpAPIError(err)
+			if apiErr != nil {
+				invalidLicense := false
+				errorMessage := ""
+				for _, apiErrItem := range apiErr.RelatedErrors {
+					if *apiErrItem.ErrorCode == nsxutil.InvalidLicenseErrorCode {
+						invalidLicense = true
+						errorMessage = *apiErrItem.ErrorMessage
+					}
+				}
+				if *apiErr.ErrorCode == nsxutil.InvalidLicenseErrorCode {
+					invalidLicense = true
+					errorMessage = *apiErr.ErrorMessage
+				}
+				if invalidLicense {
+					log.Error(err, "Invalid license, nsx-operator will restart", "error message", errorMessage)
+					os.Exit(1)
+				}
+			}
 			log.Error(err, "create or update failed, would retry exponentially", "securitypolicy", req.NamespacedName)
 			updateFail(r, &ctx, obj, &err)
 			return ResultRequeue, err

pkg/nsx/client.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 const (
@@ -265,3 +266,31 @@ func (client *Client) NSXCheckVersion(feature int) bool {
 func (client *Client) FeatureEnabled(feature int) bool {
 	return client.NSXVerChecker.featureSupported[feature] == true
 }
+
+// ValidateLicense validates NSX license. init is used to indicate whether nsx-operator is init or not
+// if not init, nsx-operator will check if license has been updated.
+// once license updated, operator will restart
+// if FeatureContainer license is false, operatore will restart
+func (client *Client) ValidateLicense(init bool) error {
+	log.Info("Checking NSX license")
+	oldContainerLicense := util.IsLicensed(util.FeatureContainer)
+	oldDfwLicense := util.IsLicensed(util.FeatureDFW)
+	err := client.NSXChecker.cluster.FetchLicense()
+	if err != nil {
+		return err
+	}
+	if !util.IsLicensed(util.FeatureContainer) {
+		err = errors.New("NSX license check failed")
+		log.Error(err, "container license is not supported")
+		return err
+	}
+	if !init {
+		newContainerLicense := util.IsLicensed(util.FeatureContainer)
+		newDfwLicense := util.IsLicensed(util.FeatureDFW)
+		if newContainerLicense != oldContainerLicense || newDfwLicense != oldDfwLicense {
+			log.Info("license updated, reset", "container license new value", newContainerLicense, "DFW license new value", newDfwLicense, "container license old value", oldContainerLicense, "DFW license old value", oldDfwLicense)
+			return errors.New("license updated")
+		}
+	}
+	return nil
+}

pkg/nsx/cluster.go

@@ -39,6 +39,12 @@ const (
 const (
 	EnvoyUrlWithCert       = "http://%s:%d/external-cert/http1/%s"
 	EnvoyUrlWithThumbprint = "http://%s:%d/external-tp/http1/%s/%s"
+	LicenseAPI             = "api/v1/licenses/licensed-features"
+)
+
+const (
+	maxNSXGetRetries = 10
+	NSXGetDelay      = 2 * time.Second
 )
 
 // Cluster consists of endpoint and provides http.Client used to send http requests.
@@ -356,16 +362,7 @@ func (cluster *Cluster) GetVersion() (*NsxVersion, error) {
 
 // HttpGet sends a http GET request to the cluster, exported for use
 func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) {
-	ep := cluster.endpoints[0]
-	serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme())
-	url = fmt.Sprintf("%s/%s", serverUrl, url)
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		log.Error(err, "failed to create http request")
-		return nil, err
-	}
-	log.V(1).Info("Get url", "url", req.URL)
-	resp, err := ep.client.Do(req)
+	resp, err := cluster.httpAction(url, "GET")
 	if err != nil {
 		log.Error(err, "failed to do http GET operation")
 		return nil, err
@@ -375,18 +372,26 @@ func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) {
 	return respJson, err
 }
 
-// HttpDelete sends a http DELETE request to the cluster, exported for use
-func (cluster *Cluster) HttpDelete(url string) error {
+func (cluster *Cluster) httpAction(url, method string) (*http.Response, error) {
 	ep := cluster.endpoints[0]
 	serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme())
 	url = fmt.Sprintf("%s/%s", serverUrl, url)
-	req, err := http.NewRequest("DELETE", url, nil)
+	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
 		log.Error(err, "failed to create http request")
-		return err
+		return nil, err
+	}
+	log.V(1).Info(method+" url", "url", req.URL)
+	resp, err := ep.client.Do(req)
+	if err != nil {
+		return nil, err
 	}
-	log.V(1).Info("Delete url", "url", req.URL)
-	_, err = ep.client.Do(req)
+	return resp, nil
+}
+
+// HttpDelete sends a http DELETE request to the cluster, exported for use
+func (cluster *Cluster) HttpDelete(url string) error {
+	_, err := cluster.httpAction(url, "DELETE")
 	if err != nil {
 		log.Error(err, "failed to do http DELETE operation")
 		return err
@@ -460,3 +465,18 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool {
 	}
 	return false
 }
+
+func (cluster *Cluster) FetchLicense() error {
+	resp, err := cluster.httpAction(LicenseAPI, "GET")
+	if err != nil {
+		log.Error(err, "failed to get nsx license")
+		return err
+	}
+	nsxLicense := &util.NsxLicense{}
+	err, _ = util.HandleHTTPResponse(resp, nsxLicense, true)
+	if err != nil {
+		return err
+	}
+	util.UpdateFeatureLicense(nsxLicense)
+	return nil
+}

pkg/nsx/cluster_test.go

@@ -4,8 +4,11 @@
 package nsx
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -17,6 +20,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 func TestNewCluster(t *testing.T) {
@@ -344,3 +348,69 @@ func TestCluster_CreateServerUrl(t *testing.T) {
 		})
 	}
 }
+
+func TestFetchLicense(t *testing.T) {
+	address := address{
+		host:   "1.2.3.4",
+		scheme: "https",
+	}
+	// Success case
+	cluster := &Cluster{endpoints: []*Endpoint{{
+		provider: &address,
+	}}}
+	cluster.config = &Config{EnvoyPort: 0}
+
+	// Request creation failure
+	patch := gomonkey.ApplyFunc(http.NewRequest,
+		func(method, url string, body io.Reader) (*http.Request, error) {
+			return nil, errors.New("request error")
+		})
+	err := cluster.FetchLicense()
+	assert.Error(t, err)
+	patch.Reset()
+
+	// HTTP error
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			return nil, errors.New("http error")
+		})
+
+	err = cluster.FetchLicense()
+	assert.Error(t, err)
+	patch.Reset()
+
+	// normal case
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			res := &nsxutil.NsxLicense{
+				Results: []struct {
+					FeatureName string `json:"feature_name"`
+					IsLicensed  bool   `json:"is_licensed"`
+				}{{
+					FeatureName: "CONTAINER",
+					IsLicensed:  true,
+				},
+					{
+						FeatureName: "DFW",
+						IsLicensed:  true,
+					},
+				},
+				ResultCount: 2,
+			}
+
+			jsonBytes, _ := json.Marshal(res)
+
+			return &http.Response{
+				StatusCode: 200,
+				Body:       io.NopCloser(bytes.NewReader(jsonBytes)),
+				Header: http.Header{
+					"Content-Type": []string{"application/json"},
+				},
+				Request: req,
+			}, nil
+		})
+	defer patch.Reset()
+	err = cluster.FetchLicense()
+	assert.Nil(t, err)
+
+}

pkg/nsx/services/securitypolicy/firewall.go

@@ -137,6 +137,10 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi
 }
 
 func (service *SecurityPolicyService) CreateOrUpdateSecurityPolicy(obj interface{}) error {
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		log.Info("no DFW license, skip creating SecurityPolicy.")
+		return nsxutil.RestrictionError{Desc: "no DFW license"}
+	}
 	var err error
 	switch obj.(type) {
 	case *networkingv1.NetworkPolicy:

pkg/nsx/services/vpc/vpc.go

@@ -625,6 +625,10 @@ func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace strin
 	if !enableAviAllowRule {
 		return nil
 	}
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		log.Info("avi rule cannot be created or updated due to no DFW license")
+		return nil
+	}
 	vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path)
 	if err != nil {
 		log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path)

pkg/nsx/services/vpc/vpc_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 var (
@@ -471,6 +472,7 @@ func TestCreateOrUpdateAVIRule(t *testing.T) {
 	sp := model.SecurityPolicy{
 		Path: &sppath1,
 	}
+	util.UpdateLicense(util.FeatureDFW, true)
 
 	// security policy not found
 	spClient.SP = sp

pkg/nsx/util/errors.go

@@ -7,6 +7,10 @@ import (
 	"fmt"
 )
 
+const (
+	InvalidLicenseErrorCode = 505
+)
+
 type NsxError interface {
 	setDetail(detail *ErrorDetail)
 	Error() string